X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/39236c6e673c41db228275375ab7fdb0f837b292..cb3231590a3c94ab4375e2228bd5e86b0cf1ad7e:/osfmk/x86_64/loose_ends.c diff --git a/osfmk/x86_64/loose_ends.c b/osfmk/x86_64/loose_ends.c index cd61326f3..807ecfc52 100644 --- a/osfmk/x86_64/loose_ends.c +++ b/osfmk/x86_64/loose_ends.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,34 +22,34 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ -/* +/* * Mach Operating System * Copyright (c) 1991,1990,1989 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ @@ -76,12 +76,20 @@ #include #include +#include #include #if !MACH_KDP #include #endif /* !MACH_KDP */ +#include + +#include +#if CONFIG_DTRACE +#include +#endif + #if 0 #undef KERNEL_DEBUG @@ -90,25 +98,28 @@ #endif +/* prevent infinite recursion when memmove calls bcopy; in string.h, bcopy is defined to call memmove */ +#undef bcopy + /* XXX - should be gone from here */ -extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys); -extern void flush_dcache64(addr64_t addr, unsigned count, int phys); -extern boolean_t phys_page_exists(ppnum_t); -extern void bcopy_no_overwrite(const char *from, char *to,vm_size_t bytes); -extern void pmap_set_reference(ppnum_t pn); -extern void mapping_set_mod(ppnum_t pa); -extern void mapping_set_ref(ppnum_t pn); - -extern void ovbcopy(const char *from, - char *to, - vm_size_t nbytes); +extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys); +extern void flush_dcache64(addr64_t addr, unsigned count, int phys); +extern boolean_t phys_page_exists(ppnum_t); +extern void bcopy_no_overwrite(const char *from, char *to, vm_size_t bytes); +extern void pmap_set_reference(ppnum_t pn); +extern void mapping_set_mod(ppnum_t pa); +extern void mapping_set_ref(ppnum_t pn); + +extern void ovbcopy(const char *from, + char *to, + vm_size_t nbytes); void machine_callstack(uintptr_t *buf, vm_size_t callstack_max); #define value_64bit(value) ((value) & 0xFFFFFFFF00000000ULL) #define low32(x) ((unsigned int)((x) & 0x00000000FFFFFFFFULL)) -#define INT_SIZE (BYTE_SIZE * sizeof (int)) +#define INT_SIZE (BYTE_SIZE * sizeof (int)) /* * Set indicated bit in bit string. @@ -145,15 +156,18 @@ ffsbit(int *s) { int offset; - for (offset = 0; !*s; offset += (int)INT_SIZE, ++s); + for (offset = 0; !*s; offset += (int)INT_SIZE, ++s) { + ; + } return offset + __builtin_ctz(*s); } int ffs(unsigned int mask) { - if (mask == 0) + if (mask == 0) { return 0; + } /* * NOTE: cannot use __builtin_ffs because it generates a call to @@ -162,18 +176,55 @@ ffs(unsigned int mask) return 1 + __builtin_ctz(mask); } +int +ffsll(unsigned long long mask) +{ + if (mask == 0) { + return 0; + } + + /* + * NOTE: cannot use __builtin_ffsll because it generates a call to + * 'ffsll' + */ + return 1 + __builtin_ctzll(mask); +} + +/* + * Find last bit set in bit string. + */ +int +fls(unsigned int mask) +{ + if (mask == 0) { + return 0; + } + + return (sizeof(mask) << 3) - __builtin_clz(mask); +} + +int +flsll(unsigned long long mask) +{ + if (mask == 0) { + return 0; + } + + return (sizeof(mask) << 3) - __builtin_clzll(mask); +} + void bzero_phys_nc( - addr64_t src64, - uint32_t bytes) + addr64_t src64, + uint32_t bytes) { - bzero_phys(src64,bytes); + bzero_phys(src64, bytes); } void bzero_phys( - addr64_t src64, - uint32_t bytes) + addr64_t src64, + uint32_t bytes) { bzero(PHYSMAP_PTOV(src64), bytes); } @@ -185,14 +236,14 @@ bzero_phys( void bcopy_phys( - addr64_t src64, - addr64_t dst64, - vm_size_t bytes) + addr64_t src64, + addr64_t dst64, + vm_size_t bytes) { /* Not necessary for K64 - but ensure we stay within a page */ - if (((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) || - ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) { - panic("bcopy_phys alignment"); + if (((((uint32_t)src64 & (NBPG - 1)) + bytes) > NBPG) || + ((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) { + panic("bcopy_phys alignment"); } bcopy(PHYSMAP_PTOV(src64), PHYSMAP_PTOV(dst64), bytes); } @@ -203,41 +254,42 @@ bcopy_phys( int apply_func_phys( - addr64_t dst64, - vm_size_t bytes, - int (*func)(void * buffer, vm_size_t bytes, void * arg), - void * arg) + addr64_t dst64, + vm_size_t bytes, + int (*func)(void * buffer, vm_size_t bytes, void * arg), + void * arg) { /* Not necessary for K64 - but ensure we stay within a page */ - if (((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) { - panic("apply_func_phys alignment"); + if (((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) { + panic("apply_func_phys alignment"); } return func(PHYSMAP_PTOV(dst64), bytes, arg); } -/* - * ovbcopy - like bcopy, but recognizes overlapping ranges and handles +/* + * ovbcopy - like bcopy, but recognizes overlapping ranges and handles * them correctly. */ void ovbcopy( - const char *from, - char *to, - vm_size_t bytes) /* num bytes to copy */ + const char *from, + char *to, + vm_size_t bytes) /* num bytes to copy */ { /* Assume that bcopy copies left-to-right (low addr first). */ - if (from + bytes <= to || to + bytes <= from || to == from) - bcopy_no_overwrite(from, to, bytes); /* non-overlapping or no-op*/ - else if (from > to) - bcopy_no_overwrite(from, to, bytes); /* overlapping but OK */ - else { + if (from + bytes <= to || to + bytes <= from || to == from) { + bcopy_no_overwrite(from, to, bytes); /* non-overlapping or no-op*/ + } else if (from > to) { + bcopy_no_overwrite(from, to, bytes); /* overlapping but OK */ + } else { /* to > from: overlapping, and must copy right-to-left. */ from += bytes - 1; to += bytes - 1; - while (bytes-- > 0) + while (bytes-- > 0) { *to-- = *from--; + } } } @@ -246,92 +298,171 @@ ovbcopy( * Read data from a physical address. Memory should not be cache inhibited. */ +uint64_t reportphyreaddelayabs; +uint64_t reportphywritedelayabs; +uint32_t reportphyreadosbt; +uint32_t reportphywriteosbt; + +#if DEVELOPMENT || DEBUG +uint32_t phyreadpanic = 1; +uint32_t phywritepanic = 1; +uint64_t tracephyreaddelayabs = 50 * NSEC_PER_USEC; +uint64_t tracephywritedelayabs = 50 * NSEC_PER_USEC; +uint64_t simulate_stretched_io = 0; +#else +uint32_t phyreadpanic = 0; +uint32_t phywritepanic = 0; +uint64_t tracephyreaddelayabs = 0; +uint64_t tracephywritedelayabs = 0; +#endif -static inline unsigned int -ml_phys_read_data(pmap_paddr_t paddr, int size) +__private_extern__ uint64_t +ml_phys_read_data(uint64_t paddr, int size) { - unsigned int result = 0; + uint64_t result = 0; + unsigned char s1; + unsigned short s2; + boolean_t istate = TRUE, timeread = FALSE; + uint64_t sabs = 0, eabs; - if (!physmap_enclosed(paddr)) + if (__improbable(!physmap_enclosed(paddr))) { panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr); + } - switch (size) { - unsigned char s1; - unsigned short s2; - case 1: + if (__improbable(reportphyreaddelayabs != 0)) { + istate = ml_set_interrupts_enabled(FALSE); + sabs = mach_absolute_time(); + timeread = TRUE; + } +#if DEVELOPMENT || DEBUG + if (__improbable(timeread && simulate_stretched_io)) { + sabs -= simulate_stretched_io; + } +#endif /* x86_64 DEVELOPMENT || DEBUG */ + + switch (size) { + case 1: s1 = *(volatile unsigned char *)PHYSMAP_PTOV(paddr); result = s1; break; - case 2: + case 2: s2 = *(volatile unsigned short *)PHYSMAP_PTOV(paddr); result = s2; break; - case 4: + case 4: result = *(volatile unsigned int *)PHYSMAP_PTOV(paddr); break; + case 8: + result = *(volatile unsigned long long *)PHYSMAP_PTOV(paddr); + break; default: - panic("Invalid size %d for ml_phys_read_data\n", size); + panic("Invalid size %d for ml_phys_read_data", size); break; - } - return result; + } + + if (__improbable(timeread == TRUE)) { + eabs = mach_absolute_time(); + +#if DEVELOPMENT || DEBUG + iotrace(IOTRACE_PHYS_READ, 0, paddr, size, result, sabs, eabs - sabs); +#endif + + if (__improbable((eabs - sabs) > reportphyreaddelayabs)) { + (void)ml_set_interrupts_enabled(istate); + + if (phyreadpanic && (machine_timeout_suspended() == FALSE)) { + panic_io_port_read(); + panic("Read from physical addr 0x%llx took %llu ns, " + "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu", + paddr, (eabs - sabs), result, sabs, eabs, + reportphyreaddelayabs); + } + + if (reportphyreadosbt) { + OSReportWithBacktrace("ml_phys_read_data took %lluus", + (eabs - sabs) / NSEC_PER_USEC); + } +#if CONFIG_DTRACE + DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs), + uint64_t, paddr, uint32_t, size, uint64_t, result); +#endif /* CONFIG_DTRACE */ + } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) { + KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ), + (eabs - sabs), sabs, paddr, result); + + (void)ml_set_interrupts_enabled(istate); + } else { + (void)ml_set_interrupts_enabled(istate); + } + } + + return result; } static unsigned long long -ml_phys_read_long_long(pmap_paddr_t paddr ) +ml_phys_read_long_long(uint64_t paddr) { - if (!physmap_enclosed(paddr)) - panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr); - return *(volatile unsigned long long *)PHYSMAP_PTOV(paddr); + return ml_phys_read_data(paddr, 8); } -unsigned int ml_phys_read( vm_offset_t paddr) +unsigned int +ml_phys_read(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 4); + return (unsigned int) ml_phys_read_data(paddr, 4); } -unsigned int ml_phys_read_word(vm_offset_t paddr) { - - return ml_phys_read_data((pmap_paddr_t)paddr, 4); +unsigned int +ml_phys_read_word(vm_offset_t paddr) +{ + return (unsigned int) ml_phys_read_data(paddr, 4); } -unsigned int ml_phys_read_64(addr64_t paddr64) +unsigned int +ml_phys_read_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return (unsigned int) ml_phys_read_data(paddr64, 4); } -unsigned int ml_phys_read_word_64(addr64_t paddr64) +unsigned int +ml_phys_read_word_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 4); + return (unsigned int) ml_phys_read_data(paddr64, 4); } -unsigned int ml_phys_read_half(vm_offset_t paddr) +unsigned int +ml_phys_read_half(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 2); + return (unsigned int) ml_phys_read_data(paddr, 2); } -unsigned int ml_phys_read_half_64(addr64_t paddr64) +unsigned int +ml_phys_read_half_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 2); + return (unsigned int) ml_phys_read_data(paddr64, 2); } -unsigned int ml_phys_read_byte(vm_offset_t paddr) +unsigned int +ml_phys_read_byte(vm_offset_t paddr) { - return ml_phys_read_data((pmap_paddr_t)paddr, 1); + return (unsigned int) ml_phys_read_data(paddr, 1); } -unsigned int ml_phys_read_byte_64(addr64_t paddr64) +unsigned int +ml_phys_read_byte_64(addr64_t paddr64) { - return ml_phys_read_data((pmap_paddr_t)paddr64, 1); + return (unsigned int) ml_phys_read_data(paddr64, 1); } -unsigned long long ml_phys_read_double(vm_offset_t paddr) +unsigned long long +ml_phys_read_double(vm_offset_t paddr) { - return ml_phys_read_long_long((pmap_paddr_t)paddr); + return ml_phys_read_long_long(paddr); } -unsigned long long ml_phys_read_double_64(addr64_t paddr64) +unsigned long long +ml_phys_read_double_64(addr64_t paddr64) { - return ml_phys_read_long_long((pmap_paddr_t)paddr64); + return ml_phys_read_long_long(paddr64); } @@ -340,87 +471,323 @@ unsigned long long ml_phys_read_double_64(addr64_t paddr64) * Write data to a physical address. Memory should not be cache inhibited. */ -static inline void -ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size) +__private_extern__ void +ml_phys_write_data(uint64_t paddr, unsigned long long data, int size) { - if (!physmap_enclosed(paddr)) + boolean_t istate = TRUE, timewrite = FALSE; + uint64_t sabs = 0, eabs; + + if (__improbable(!physmap_enclosed(paddr))) { panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr); + } - switch (size) { - case 1: - *(volatile unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data; - break; - case 2: - *(volatile unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data; - break; - case 4: - *(volatile unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data; - break; + if (__improbable(reportphywritedelayabs != 0)) { + istate = ml_set_interrupts_enabled(FALSE); + sabs = mach_absolute_time(); + timewrite = TRUE; + } +#if DEVELOPMENT || DEBUG + if (__improbable(timewrite && simulate_stretched_io)) { + sabs -= simulate_stretched_io; + } +#endif /* x86_64 DEVELOPMENT || DEBUG */ + + switch (size) { + case 1: + *(volatile unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data; + break; + case 4: + *(volatile unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data; + break; + case 8: + *(volatile unsigned long *)PHYSMAP_PTOV(paddr) = data; + break; default: - panic("Invalid size %d for ml_phys_write_data\n", size); + panic("Invalid size %d for ml_phys_write_data", size); break; - } + } + + if (__improbable(timewrite == TRUE)) { + eabs = mach_absolute_time(); + +#if DEVELOPMENT || DEBUG + iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs); +#endif + + if (__improbable((eabs - sabs) > reportphywritedelayabs)) { + (void)ml_set_interrupts_enabled(istate); + + if (phywritepanic && (machine_timeout_suspended() == FALSE)) { + panic_io_port_read(); + panic("Write to physical addr 0x%llx took %llu ns, " + "data: 0x%llx (start: %llu, end: %llu), ceiling: %llu", + paddr, (eabs - sabs), data, sabs, eabs, + reportphywritedelayabs); + } + + if (reportphywriteosbt) { + OSReportWithBacktrace("ml_phys_write_data (%p, 0x%llx) " + "took %lluus", + paddr, data, (eabs - sabs) / NSEC_PER_USEC); + } +#if CONFIG_DTRACE + DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs), + uint64_t, paddr, uint32_t, size, uint64_t, data); +#endif /* CONFIG_DTRACE */ + } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) { + KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE), + (eabs - sabs), sabs, paddr, data); + + (void)ml_set_interrupts_enabled(istate); + } else { + (void)ml_set_interrupts_enabled(istate); + } + } } -static void -ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) +void +ml_phys_write_byte(vm_offset_t paddr, unsigned int data) { - if (!physmap_enclosed(paddr)) - panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr); + ml_phys_write_data(paddr, data, 1); +} - *(volatile unsigned long long *)PHYSMAP_PTOV(paddr) = data; +void +ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data(paddr64, data, 1); } -void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_half(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 1); + ml_phys_write_data(paddr, data, 2); } -void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_half_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); + ml_phys_write_data(paddr64, data, 2); } -void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +void +ml_phys_write(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 2); + ml_phys_write_data(paddr, data, 4); } -void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); + ml_phys_write_data(paddr64, data, 4); } -void ml_phys_write(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_word(vm_offset_t paddr, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data(paddr, data, 4); } -void ml_phys_write_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_word_64(addr64_t paddr64, unsigned int data) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + ml_phys_write_data(paddr64, data, 4); } -void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +void +ml_phys_write_double(vm_offset_t paddr, unsigned long long data) { - ml_phys_write_data((pmap_paddr_t)paddr, data, 4); + ml_phys_write_data(paddr, data, 8); } -void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +void +ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +{ + ml_phys_write_data(paddr64, data, 8); +} + +uint32_t +ml_port_io_read(uint16_t ioport, int size) +{ + uint32_t result = 0; + + uint64_t sabs, eabs; + boolean_t istate, timeread = FALSE; + + if (__improbable(reportphyreaddelayabs != 0)) { + istate = ml_set_interrupts_enabled(FALSE); + sabs = mach_absolute_time(); + timeread = TRUE; + } + +#if DEVELOPMENT || DEBUG + if (__improbable(timeread && simulate_stretched_io)) { + sabs -= simulate_stretched_io; + } +#endif /* x86_64 DEVELOPMENT || DEBUG */ + + switch (size) { + case 1: + result = inb(ioport); + break; + case 2: + result = inw(ioport); + break; + case 4: + result = inl(ioport); + break; + default: + panic("Invalid size %d for ml_port_io_read(0x%x)", size, (unsigned)ioport); + break; + } + + if (__improbable(timeread == TRUE)) { + eabs = mach_absolute_time(); + +#if DEVELOPMENT || DEBUG + iotrace(IOTRACE_PORTIO_READ, 0, ioport, size, result, sabs, eabs - sabs); +#endif + + if (__improbable((eabs - sabs) > reportphyreaddelayabs)) { + (void)ml_set_interrupts_enabled(istate); + + if (phyreadpanic && (machine_timeout_suspended() == FALSE)) { + panic_io_port_read(); + panic("Read from IO port 0x%x took %llu ns, " + "result: 0x%x (start: %llu, end: %llu), ceiling: %llu", + ioport, (eabs - sabs), result, sabs, eabs, + reportphyreaddelayabs); + } + + if (reportphyreadosbt) { + OSReportWithBacktrace("ml_port_io_read(0x%x) took %lluus", + ioport, (eabs - sabs) / NSEC_PER_USEC); + } +#if CONFIG_DTRACE + DTRACE_PHYSLAT3(portioread, uint64_t, (eabs - sabs), + uint16_t, ioport, uint32_t, size); +#endif /* CONFIG_DTRACE */ + } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) { + KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_READ), + (eabs - sabs), sabs, ioport, result); + + (void)ml_set_interrupts_enabled(istate); + } else { + (void)ml_set_interrupts_enabled(istate); + } + } + + return result; +} + +void +ml_port_io_write(uint16_t ioport, uint32_t val, int size) +{ + uint64_t sabs, eabs; + boolean_t istate, timewrite = FALSE; + + if (__improbable(reportphywritedelayabs != 0)) { + istate = ml_set_interrupts_enabled(FALSE); + sabs = mach_absolute_time(); + timewrite = TRUE; + } +#if DEVELOPMENT || DEBUG + if (__improbable(timewrite && simulate_stretched_io)) { + sabs -= simulate_stretched_io; + } +#endif /* x86_64 DEVELOPMENT || DEBUG */ + + switch (size) { + case 1: + outb(ioport, (uint8_t)val); + break; + case 2: + outw(ioport, (uint16_t)val); + break; + case 4: + outl(ioport, (uint32_t)val); + break; + default: + panic("Invalid size %d for ml_port_io_write(0x%x)", size, (unsigned)ioport); + break; + } + + if (__improbable(timewrite == TRUE)) { + eabs = mach_absolute_time(); + +#if DEVELOPMENT || DEBUG + iotrace(IOTRACE_PORTIO_WRITE, 0, ioport, size, val, sabs, eabs - sabs); +#endif + + if (__improbable((eabs - sabs) > reportphywritedelayabs)) { + (void)ml_set_interrupts_enabled(istate); + + if (phywritepanic && (machine_timeout_suspended() == FALSE)) { + panic_io_port_read(); + panic("Write to IO port 0x%x took %llu ns, val: 0x%x" + " (start: %llu, end: %llu), ceiling: %llu", + ioport, (eabs - sabs), val, sabs, eabs, + reportphywritedelayabs); + } + + if (reportphywriteosbt) { + OSReportWithBacktrace("ml_port_io_write(0x%x, %d, 0x%llx) " + "took %lluus", + ioport, size, val, (eabs - sabs) / NSEC_PER_USEC); + } + +#if CONFIG_DTRACE + DTRACE_PHYSLAT4(portiowrite, uint64_t, (eabs - sabs), + uint16_t, ioport, uint32_t, size, uint64_t, val); +#endif /* CONFIG_DTRACE */ + } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) { + KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_WRITE), + (eabs - sabs), sabs, ioport, val); + + (void)ml_set_interrupts_enabled(istate); + } else { + (void)ml_set_interrupts_enabled(istate); + } + } +} + +uint8_t +ml_port_io_read8(uint16_t ioport) +{ + return ml_port_io_read(ioport, 1); +} + +uint16_t +ml_port_io_read16(uint16_t ioport) { - ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); + return ml_port_io_read(ioport, 2); } -void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +uint32_t +ml_port_io_read32(uint16_t ioport) +{ + return ml_port_io_read(ioport, 4); +} + +void +ml_port_io_write8(uint16_t ioport, uint8_t val) { - ml_phys_write_long_long((pmap_paddr_t)paddr, data); + ml_port_io_write(ioport, val, 1); } -void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +void +ml_port_io_write16(uint16_t ioport, uint16_t val) { - ml_phys_write_long_long((pmap_paddr_t)paddr64, data); + ml_port_io_write(ioport, val, 2); } +void +ml_port_io_write32(uint16_t ioport, uint32_t val) +{ + ml_port_io_write(ioport, val, 4); +} /* PCI config cycle probing * @@ -433,12 +800,13 @@ void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) boolean_t ml_probe_read(vm_offset_t paddr, unsigned int *val) { - if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4) - return FALSE; + if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4) { + return FALSE; + } - *val = ml_phys_read((pmap_paddr_t)paddr); + *val = ml_phys_read(paddr); - return TRUE; + return TRUE; } /* @@ -447,36 +815,42 @@ ml_probe_read(vm_offset_t paddr, unsigned int *val) * have a machine check here. So we have to be able to handle that. * We assume that machine checks are enabled both in MSR and HIDs */ -boolean_t +boolean_t ml_probe_read_64(addr64_t paddr64, unsigned int *val) { - if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4) - return FALSE; + if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4) { + return FALSE; + } - *val = ml_phys_read_64((pmap_paddr_t)paddr64); - return TRUE; + *val = ml_phys_read_64(paddr64); + return TRUE; } -int bcmp( - const void *pa, - const void *pb, - size_t len) +#undef bcmp +int +bcmp( + const void *pa, + const void *pb, + size_t len) { const char *a = (const char *)pa; const char *b = (const char *)pb; - if (len == 0) + if (len == 0) { return 0; + } - do - if (*a++ != *b++) + do{ + if (*a++ != *b++) { break; - while (--len); + } + } while (--len); return (int)len; } +#undef memcmp int memcmp(const void *s1, const void *s2, size_t n) { @@ -484,11 +858,20 @@ memcmp(const void *s1, const void *s2, size_t n) const unsigned char *p1 = s1, *p2 = s2; do { - if (*p1++ != *p2++) - return (*--p1 - *--p2); + if (*p1++ != *p2++) { + return *--p1 - *--p2; + } } while (--n != 0); } - return (0); + return 0; +} + +#undef memmove +void * +memmove(void *dst, const void *src, size_t ulen) +{ + bcopy(src, dst, ulen); + return dst; } /* @@ -497,57 +880,52 @@ memcmp(const void *s1, const void *s2, size_t n) * the terminating null character. */ +#undef strlen size_t strlen( - register const char *string) + const char *string) { - register const char *ret = string; + const char *ret = string; - while (*string++ != '\0') + while (*string++ != '\0') { continue; + } return string - 1 - ret; } -uint32_t -hw_compare_and_store(uint32_t oldval, uint32_t newval, volatile uint32_t *dest) -{ - return OSCompareAndSwap((UInt32)oldval, - (UInt32)newval, - (volatile UInt32 *)dest); -} - -#if MACH_ASSERT +#if MACH_ASSERT /* * Machine-dependent routine to fill in an array with up to callstack_max * levels of return pc information. */ -void machine_callstack( - __unused uintptr_t *buf, - __unused vm_size_t callstack_max) +void +machine_callstack( + __unused uintptr_t *buf, + __unused vm_size_t callstack_max) { } -#endif /* MACH_ASSERT */ +#endif /* MACH_ASSERT */ -void fillPage(ppnum_t pa, unsigned int fill) +void +fillPage(ppnum_t pa, unsigned int fill) { - pmap_paddr_t src; - int i; + uint64_t src; int cnt = PAGE_SIZE / sizeof(unsigned int); - unsigned int *addr; src = i386_ptob(pa); - for (i = 0, addr = (unsigned int *)PHYSMAP_PTOV(src); i < cnt; i++) - *addr++ = fill; + memset_word((int *)PHYSMAP_PTOV(src), fill, cnt); } -static inline void __clflush(void *ptr) +static inline void +__clflush(void *ptr) { - __asm__ volatile("clflush (%0)" : : "r" (ptr)); + __asm__ volatile ("clflush (%0)" : : "r" (ptr)); } -void dcache_incoherent_io_store64(addr64_t pa, unsigned int count) +void +dcache_incoherent_io_store64(addr64_t pa, unsigned int count) { addr64_t linesize = cpuid_info()->cache_linesize; addr64_t bound = (pa + count + linesize - 1) & ~(linesize - 1); @@ -562,9 +940,10 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count) mfence(); } -void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count) +void +dcache_incoherent_io_flush64(addr64_t pa, unsigned int count) { - return(dcache_incoherent_io_store64(pa,count)); + return dcache_incoherent_io_store64(pa, count); } void @@ -572,10 +951,9 @@ flush_dcache64(addr64_t addr, unsigned count, int phys) { if (phys) { dcache_incoherent_io_flush64(addr, count); - } - else { + } else { uint64_t linesize = cpuid_info()->cache_linesize; - addr64_t bound = (addr + count + linesize -1) & ~(linesize - 1); + addr64_t bound = (addr + count + linesize - 1) & ~(linesize - 1); mfence(); while (addr < bound) { __clflush((void *) (uintptr_t) addr); @@ -587,8 +965,8 @@ flush_dcache64(addr64_t addr, unsigned count, int phys) void invalidate_icache64(__unused addr64_t addr, - __unused unsigned count, - __unused int phys) + __unused unsigned count, + __unused int phys) { } @@ -598,37 +976,38 @@ addr64_t vm_last_addr; void mapping_set_mod(ppnum_t pn) { - pmap_set_modify(pn); + pmap_set_modify(pn); } void mapping_set_ref(ppnum_t pn) { - pmap_set_reference(pn); + pmap_set_reference(pn); } -extern i386_cpu_info_t cpuid_cpu_info; +extern i386_cpu_info_t cpuid_cpu_info; void cache_flush_page_phys(ppnum_t pa) { - boolean_t istate; - unsigned char *cacheline_addr; - i386_cpu_info_t *cpuid_infop = cpuid_info(); - int cacheline_size; - int cachelines_to_flush; + boolean_t istate; + unsigned char *cacheline_addr; + i386_cpu_info_t *cpuid_infop = cpuid_info(); + int cacheline_size; + int cachelines_to_flush; cacheline_size = cpuid_infop->cache_linesize; - if (cacheline_size == 0) + if (cacheline_size == 0) { panic("cacheline_size=0 cpuid_infop=%p\n", cpuid_infop); - cachelines_to_flush = PAGE_SIZE/cacheline_size; + } + cachelines_to_flush = PAGE_SIZE / cacheline_size; mfence(); istate = ml_set_interrupts_enabled(FALSE); for (cacheline_addr = (unsigned char *)PHYSMAP_PTOV(i386_ptob(pa)); - cachelines_to_flush > 0; - cachelines_to_flush--, cacheline_addr += cacheline_size) { + cachelines_to_flush > 0; + cachelines_to_flush--, cacheline_addr += cacheline_size) { __clflush((void *) cacheline_addr); } @@ -646,27 +1025,15 @@ kdp_register_callout(kdp_callout_fn_t fn, void *arg) } #endif -/* - * Return a uniformly distributed 64-bit random number. - * - * This interface should have minimal dependencies on kernel - * services, and thus be available very early in the life - * of the kernel. But as a result, it may not be very random - * on all platforms. - */ -uint64_t -early_random(void) -{ - return (ml_early_random()); -} - #if !CONFIG_VMX -int host_vmxon(boolean_t exclusive __unused) +int +host_vmxon(boolean_t exclusive __unused) { return VMX_UNSUPPORTED; } -void host_vmxoff(void) +void +host_vmxoff(void) { return; }