/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
-/*
+/*
* Mach Operating System
* Copyright (c) 1991,1990,1989 Carnegie Mellon University
* All Rights Reserved.
- *
+ *
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
- *
+ *
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- *
+ *
* Carnegie Mellon requests users of this software to return to
- *
+ *
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
- *
+ *
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
#include <vm/vm_fault.h>
#include <libkern/OSAtomic.h>
+#include <libkern/OSDebug.h>
#include <sys/kdebug.h>
+#if !MACH_KDP
+#include <kdp/kdp_callout.h>
+#endif /* !MACH_KDP */
+
+#include <architecture/i386/pio.h>
+
+#include <libkern/OSDebug.h>
+#if CONFIG_DTRACE
+#include <mach/sdt.h>
+#endif
+
#if 0
#undef KERNEL_DEBUG
#endif
+/* prevent infinite recursion when memmove calls bcopy; in string.h, bcopy is defined to call memmove */
+#undef bcopy
+
/* XXX - should be gone from here */
-extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
-extern void flush_dcache64(addr64_t addr, unsigned count, int phys);
-extern boolean_t phys_page_exists(ppnum_t);
-extern void bcopy_no_overwrite(const char *from, char *to,vm_size_t bytes);
-extern void pmap_set_reference(ppnum_t pn);
-extern void mapping_set_mod(ppnum_t pa);
-extern void mapping_set_ref(ppnum_t pn);
+extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
+extern void flush_dcache64(addr64_t addr, unsigned count, int phys);
+extern boolean_t phys_page_exists(ppnum_t);
+extern void bcopy_no_overwrite(const char *from, char *to, vm_size_t bytes);
+extern void pmap_set_reference(ppnum_t pn);
+extern void mapping_set_mod(ppnum_t pa);
+extern void mapping_set_ref(ppnum_t pn);
-extern void ovbcopy(const char *from,
- char *to,
- vm_size_t nbytes);
-void machine_callstack(natural_t *buf, vm_size_t callstack_max);
+extern void ovbcopy(const char *from,
+ char *to,
+ vm_size_t nbytes);
+void machine_callstack(uintptr_t *buf, vm_size_t callstack_max);
#define value_64bit(value) ((value) & 0xFFFFFFFF00000000ULL)
#define low32(x) ((unsigned int)((x) & 0x00000000FFFFFFFFULL))
-#define INT_SIZE (BYTE_SIZE * sizeof (int))
+#define INT_SIZE (BYTE_SIZE * sizeof (int))
/*
* Set indicated bit in bit string.
{
int offset;
- for (offset = 0; !*s; offset += (int)INT_SIZE, ++s);
+ for (offset = 0; !*s; offset += (int)INT_SIZE, ++s) {
+ ;
+ }
return offset + __builtin_ctz(*s);
}
int
ffs(unsigned int mask)
{
- if (mask == 0)
+ if (mask == 0) {
return 0;
+ }
/*
* NOTE: cannot use __builtin_ffs because it generates a call to
return 1 + __builtin_ctz(mask);
}
+int
+ffsll(unsigned long long mask)
+{
+ if (mask == 0) {
+ return 0;
+ }
+
+ /*
+ * NOTE: cannot use __builtin_ffsll because it generates a call to
+ * 'ffsll'
+ */
+ return 1 + __builtin_ctzll(mask);
+}
+
+/*
+ * Find last bit set in bit string.
+ */
+int
+fls(unsigned int mask)
+{
+ if (mask == 0) {
+ return 0;
+ }
+
+ return (sizeof(mask) << 3) - __builtin_clz(mask);
+}
+
+int
+flsll(unsigned long long mask)
+{
+ if (mask == 0) {
+ return 0;
+ }
+
+ return (sizeof(mask) << 3) - __builtin_clzll(mask);
+}
+
void
bzero_phys_nc(
- addr64_t src64,
- uint32_t bytes)
+ addr64_t src64,
+ uint32_t bytes)
{
- bzero_phys(src64,bytes);
+ bzero_phys(src64, bytes);
}
void
bzero_phys(
- addr64_t src64,
- uint32_t bytes)
+ addr64_t src64,
+ uint32_t bytes)
{
bzero(PHYSMAP_PTOV(src64), bytes);
}
void
bcopy_phys(
- addr64_t src64,
- addr64_t dst64,
- vm_size_t bytes)
+ addr64_t src64,
+ addr64_t dst64,
+ vm_size_t bytes)
{
/* Not necessary for K64 - but ensure we stay within a page */
- if (((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) ||
- ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
- panic("bcopy_phys alignment");
+ if (((((uint32_t)src64 & (NBPG - 1)) + bytes) > NBPG) ||
+ ((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) {
+ panic("bcopy_phys alignment");
}
bcopy(PHYSMAP_PTOV(src64), PHYSMAP_PTOV(dst64), bytes);
}
int
apply_func_phys(
- addr64_t dst64,
- vm_size_t bytes,
- int (*func)(void * buffer, vm_size_t bytes, void * arg),
- void * arg)
+ addr64_t dst64,
+ vm_size_t bytes,
+ int (*func)(void * buffer, vm_size_t bytes, void * arg),
+ void * arg)
{
/* Not necessary for K64 - but ensure we stay within a page */
- if (((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
- panic("apply_func_phys alignment");
+ if (((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) {
+ panic("apply_func_phys alignment");
}
return func(PHYSMAP_PTOV(dst64), bytes, arg);
}
-/*
- * ovbcopy - like bcopy, but recognizes overlapping ranges and handles
+/*
+ * ovbcopy - like bcopy, but recognizes overlapping ranges and handles
* them correctly.
*/
void
ovbcopy(
- const char *from,
- char *to,
- vm_size_t bytes) /* num bytes to copy */
+ const char *from,
+ char *to,
+ vm_size_t bytes) /* num bytes to copy */
{
/* Assume that bcopy copies left-to-right (low addr first). */
- if (from + bytes <= to || to + bytes <= from || to == from)
- bcopy_no_overwrite(from, to, bytes); /* non-overlapping or no-op*/
- else if (from > to)
- bcopy_no_overwrite(from, to, bytes); /* overlapping but OK */
- else {
+ if (from + bytes <= to || to + bytes <= from || to == from) {
+ bcopy_no_overwrite(from, to, bytes); /* non-overlapping or no-op*/
+ } else if (from > to) {
+ bcopy_no_overwrite(from, to, bytes); /* overlapping but OK */
+ } else {
/* to > from: overlapping, and must copy right-to-left. */
from += bytes - 1;
to += bytes - 1;
- while (bytes-- > 0)
+ while (bytes-- > 0) {
*to-- = *from--;
+ }
}
}
* Read data from a physical address. Memory should not be cache inhibited.
*/
+uint64_t reportphyreaddelayabs;
+uint64_t reportphywritedelayabs;
+uint32_t reportphyreadosbt;
+uint32_t reportphywriteosbt;
+
+#if DEVELOPMENT || DEBUG
+uint32_t phyreadpanic = 1;
+uint32_t phywritepanic = 1;
+uint64_t tracephyreaddelayabs = 50 * NSEC_PER_USEC;
+uint64_t tracephywritedelayabs = 50 * NSEC_PER_USEC;
+uint64_t simulate_stretched_io = 0;
+#else
+uint32_t phyreadpanic = 0;
+uint32_t phywritepanic = 0;
+uint64_t tracephyreaddelayabs = 0;
+uint64_t tracephywritedelayabs = 0;
+#endif
-static inline unsigned int
-ml_phys_read_data(pmap_paddr_t paddr, int size)
+__private_extern__ uint64_t
+ml_phys_read_data(uint64_t paddr, int size)
{
- unsigned int result;
+ uint64_t result = 0;
+ unsigned char s1;
+ unsigned short s2;
+ boolean_t istate = TRUE, timeread = FALSE;
+ uint64_t sabs = 0, eabs;
+
+ if (__improbable(!physmap_enclosed(paddr))) {
+ panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
+ }
+
+ if (__improbable(reportphyreaddelayabs != 0)) {
+ istate = ml_set_interrupts_enabled(FALSE);
+ sabs = mach_absolute_time();
+ timeread = TRUE;
+ }
+#if DEVELOPMENT || DEBUG
+ if (__improbable(timeread && simulate_stretched_io)) {
+ sabs -= simulate_stretched_io;
+ }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+ switch (size) {
+ case 1:
+ s1 = *(volatile unsigned char *)PHYSMAP_PTOV(paddr);
+ result = s1;
+ break;
+ case 2:
+ s2 = *(volatile unsigned short *)PHYSMAP_PTOV(paddr);
+ result = s2;
+ break;
+ case 4:
+ result = *(volatile unsigned int *)PHYSMAP_PTOV(paddr);
+ break;
+ case 8:
+ result = *(volatile unsigned long long *)PHYSMAP_PTOV(paddr);
+ break;
+ default:
+ panic("Invalid size %d for ml_phys_read_data", size);
+ break;
+ }
- switch (size) {
- unsigned char s1;
- unsigned short s2;
- case 1:
- s1 = *(unsigned char *)PHYSMAP_PTOV(paddr);
- result = s1;
- break;
- case 2:
- s2 = *(unsigned short *)PHYSMAP_PTOV(paddr);
- result = s2;
- break;
- case 4:
- default:
- result = *(unsigned int *)PHYSMAP_PTOV(paddr);
- break;
- }
+ if (__improbable(timeread == TRUE)) {
+ eabs = mach_absolute_time();
- return result;
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_PHYS_READ, 0, paddr, size, result, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
+ panic_io_port_read();
+ panic("Read from physical addr 0x%llx took %llu ns, "
+ "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
+ paddr, (eabs - sabs), result, sabs, eabs,
+ reportphyreaddelayabs);
+ }
+
+ if (reportphyreadosbt) {
+ OSReportWithBacktrace("ml_phys_read_data took %lluus",
+ (eabs - sabs) / NSEC_PER_USEC);
+ }
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs),
+ uint64_t, paddr, uint32_t, size, uint64_t, result);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) {
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
+ (eabs - sabs), sabs, paddr, result);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
+
+ return result;
}
static unsigned long long
-ml_phys_read_long_long(pmap_paddr_t paddr )
+ml_phys_read_long_long(uint64_t paddr)
{
- return *(unsigned long long *)PHYSMAP_PTOV(paddr);
+ return ml_phys_read_data(paddr, 8);
}
-unsigned int ml_phys_read( vm_offset_t paddr)
+unsigned int
+ml_phys_read(vm_offset_t paddr)
{
- return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+ return (unsigned int) ml_phys_read_data(paddr, 4);
}
-unsigned int ml_phys_read_word(vm_offset_t paddr) {
-
- return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+unsigned int
+ml_phys_read_word(vm_offset_t paddr)
+{
+ return (unsigned int) ml_phys_read_data(paddr, 4);
}
-unsigned int ml_phys_read_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_64(addr64_t paddr64)
{
- return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+ return (unsigned int) ml_phys_read_data(paddr64, 4);
}
-unsigned int ml_phys_read_word_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_word_64(addr64_t paddr64)
{
- return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+ return (unsigned int) ml_phys_read_data(paddr64, 4);
}
-unsigned int ml_phys_read_half(vm_offset_t paddr)
+unsigned int
+ml_phys_read_half(vm_offset_t paddr)
{
- return ml_phys_read_data((pmap_paddr_t)paddr, 2);
+ return (unsigned int) ml_phys_read_data(paddr, 2);
}
-unsigned int ml_phys_read_half_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_half_64(addr64_t paddr64)
{
- return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
+ return (unsigned int) ml_phys_read_data(paddr64, 2);
}
-unsigned int ml_phys_read_byte(vm_offset_t paddr)
+unsigned int
+ml_phys_read_byte(vm_offset_t paddr)
{
- return ml_phys_read_data((pmap_paddr_t)paddr, 1);
+ return (unsigned int) ml_phys_read_data(paddr, 1);
}
-unsigned int ml_phys_read_byte_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_byte_64(addr64_t paddr64)
{
- return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
+ return (unsigned int) ml_phys_read_data(paddr64, 1);
}
-unsigned long long ml_phys_read_double(vm_offset_t paddr)
+unsigned long long
+ml_phys_read_double(vm_offset_t paddr)
{
- return ml_phys_read_long_long((pmap_paddr_t)paddr);
+ return ml_phys_read_long_long(paddr);
}
-unsigned long long ml_phys_read_double_64(addr64_t paddr64)
+unsigned long long
+ml_phys_read_double_64(addr64_t paddr64)
{
- return ml_phys_read_long_long((pmap_paddr_t)paddr64);
+ return ml_phys_read_long_long(paddr64);
}
* Write data to a physical address. Memory should not be cache inhibited.
*/
-static inline void
-ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
+__private_extern__ void
+ml_phys_write_data(uint64_t paddr, unsigned long long data, int size)
+{
+ boolean_t istate = TRUE, timewrite = FALSE;
+ uint64_t sabs = 0, eabs;
+
+ if (__improbable(!physmap_enclosed(paddr))) {
+ panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
+ }
+
+ if (__improbable(reportphywritedelayabs != 0)) {
+ istate = ml_set_interrupts_enabled(FALSE);
+ sabs = mach_absolute_time();
+ timewrite = TRUE;
+ }
+#if DEVELOPMENT || DEBUG
+ if (__improbable(timewrite && simulate_stretched_io)) {
+ sabs -= simulate_stretched_io;
+ }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+ switch (size) {
+ case 1:
+ *(volatile unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data;
+ break;
+ case 2:
+ *(volatile unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data;
+ break;
+ case 4:
+ *(volatile unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data;
+ break;
+ case 8:
+ *(volatile unsigned long *)PHYSMAP_PTOV(paddr) = data;
+ break;
+ default:
+ panic("Invalid size %d for ml_phys_write_data", size);
+ break;
+ }
+
+ if (__improbable(timewrite == TRUE)) {
+ eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphywritedelayabs)) {
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phywritepanic && (machine_timeout_suspended() == FALSE)) {
+ panic_io_port_read();
+ panic("Write to physical addr 0x%llx took %llu ns, "
+ "data: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
+ paddr, (eabs - sabs), data, sabs, eabs,
+ reportphywritedelayabs);
+ }
+
+ if (reportphywriteosbt) {
+ OSReportWithBacktrace("ml_phys_write_data (%p, 0x%llx) "
+ "took %lluus",
+ paddr, data, (eabs - sabs) / NSEC_PER_USEC);
+ }
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs),
+ uint64_t, paddr, uint32_t, size, uint64_t, data);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) {
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
+ (eabs - sabs), sabs, paddr, data);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
+}
+
+void
+ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
+{
+ ml_phys_write_data(paddr, data, 1);
+}
+
+void
+ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
+{
+ ml_phys_write_data(paddr64, data, 1);
+}
+
+void
+ml_phys_write_half(vm_offset_t paddr, unsigned int data)
{
- switch (size) {
- case 1:
- *(unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data;
- break;
- case 2:
- *(unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data;
- break;
- case 4:
- default:
- *(unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data;
- break;
- }
+ ml_phys_write_data(paddr, data, 2);
}
-static void
-ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
+void
+ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
{
- *(unsigned long long *)PHYSMAP_PTOV(paddr) = data;
+ ml_phys_write_data(paddr64, data, 2);
}
-void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write(vm_offset_t paddr, unsigned int data)
{
- ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
+ ml_phys_write_data(paddr, data, 4);
}
-void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_64(addr64_t paddr64, unsigned int data)
{
- ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
+ ml_phys_write_data(paddr64, data, 4);
}
-void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_word(vm_offset_t paddr, unsigned int data)
{
- ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
+ ml_phys_write_data(paddr, data, 4);
}
-void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
{
- ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
+ ml_phys_write_data(paddr64, data, 4);
}
-void ml_phys_write(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
{
- ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+ ml_phys_write_data(paddr, data, 8);
}
-void ml_phys_write_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
{
- ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+ ml_phys_write_data(paddr64, data, 8);
}
-void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
+uint32_t
+ml_port_io_read(uint16_t ioport, int size)
{
- ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+ uint32_t result = 0;
+
+ uint64_t sabs, eabs;
+ boolean_t istate, timeread = FALSE;
+
+ if (__improbable(reportphyreaddelayabs != 0)) {
+ istate = ml_set_interrupts_enabled(FALSE);
+ sabs = mach_absolute_time();
+ timeread = TRUE;
+ }
+
+#if DEVELOPMENT || DEBUG
+ if (__improbable(timeread && simulate_stretched_io)) {
+ sabs -= simulate_stretched_io;
+ }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+ switch (size) {
+ case 1:
+ result = inb(ioport);
+ break;
+ case 2:
+ result = inw(ioport);
+ break;
+ case 4:
+ result = inl(ioport);
+ break;
+ default:
+ panic("Invalid size %d for ml_port_io_read(0x%x)", size, (unsigned)ioport);
+ break;
+ }
+
+ if (__improbable(timeread == TRUE)) {
+ eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_PORTIO_READ, 0, ioport, size, result, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
+ panic_io_port_read();
+ panic("Read from IO port 0x%x took %llu ns, "
+ "result: 0x%x (start: %llu, end: %llu), ceiling: %llu",
+ ioport, (eabs - sabs), result, sabs, eabs,
+ reportphyreaddelayabs);
+ }
+
+ if (reportphyreadosbt) {
+ OSReportWithBacktrace("ml_port_io_read(0x%x) took %lluus",
+ ioport, (eabs - sabs) / NSEC_PER_USEC);
+ }
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT3(portioread, uint64_t, (eabs - sabs),
+ uint16_t, ioport, uint32_t, size);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) {
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_READ),
+ (eabs - sabs), sabs, ioport, result);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
+
+ return result;
+}
+
+void
+ml_port_io_write(uint16_t ioport, uint32_t val, int size)
+{
+ uint64_t sabs, eabs;
+ boolean_t istate, timewrite = FALSE;
+
+ if (__improbable(reportphywritedelayabs != 0)) {
+ istate = ml_set_interrupts_enabled(FALSE);
+ sabs = mach_absolute_time();
+ timewrite = TRUE;
+ }
+#if DEVELOPMENT || DEBUG
+ if (__improbable(timewrite && simulate_stretched_io)) {
+ sabs -= simulate_stretched_io;
+ }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+ switch (size) {
+ case 1:
+ outb(ioport, (uint8_t)val);
+ break;
+ case 2:
+ outw(ioport, (uint16_t)val);
+ break;
+ case 4:
+ outl(ioport, (uint32_t)val);
+ break;
+ default:
+ panic("Invalid size %d for ml_port_io_write(0x%x)", size, (unsigned)ioport);
+ break;
+ }
+
+ if (__improbable(timewrite == TRUE)) {
+ eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_PORTIO_WRITE, 0, ioport, size, val, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphywritedelayabs)) {
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phywritepanic && (machine_timeout_suspended() == FALSE)) {
+ panic_io_port_read();
+ panic("Write to IO port 0x%x took %llu ns, val: 0x%x"
+ " (start: %llu, end: %llu), ceiling: %llu",
+ ioport, (eabs - sabs), val, sabs, eabs,
+ reportphywritedelayabs);
+ }
+
+ if (reportphywriteosbt) {
+ OSReportWithBacktrace("ml_port_io_write(0x%x, %d, 0x%llx) "
+ "took %lluus",
+ ioport, size, val, (eabs - sabs) / NSEC_PER_USEC);
+ }
+
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT4(portiowrite, uint64_t, (eabs - sabs),
+ uint16_t, ioport, uint32_t, size, uint64_t, val);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) {
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_WRITE),
+ (eabs - sabs), sabs, ioport, val);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
}
-void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
+uint8_t
+ml_port_io_read8(uint16_t ioport)
{
- ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+ return ml_port_io_read(ioport, 1);
}
-void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
+uint16_t
+ml_port_io_read16(uint16_t ioport)
{
- ml_phys_write_long_long((pmap_paddr_t)paddr, data);
+ return ml_port_io_read(ioport, 2);
}
-void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
+uint32_t
+ml_port_io_read32(uint16_t ioport)
{
- ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
+ return ml_port_io_read(ioport, 4);
}
+void
+ml_port_io_write8(uint16_t ioport, uint8_t val)
+{
+ ml_port_io_write(ioport, val, 1);
+}
+
+void
+ml_port_io_write16(uint16_t ioport, uint16_t val)
+{
+ ml_port_io_write(ioport, val, 2);
+}
+
+void
+ml_port_io_write32(uint16_t ioport, uint32_t val)
+{
+ ml_port_io_write(ioport, val, 4);
+}
/* PCI config cycle probing
*
*
* Read the memory location at physical address paddr.
- * This is a part of a device probe, so there is a good chance we will
- * have a machine check here. So we have to be able to handle that.
- * We assume that machine checks are enabled both in MSR and HIDs
+ * *Does not* recover from machine checks, unlike the PowerPC implementation.
+ * Should probably be deprecated.
*/
boolean_t
ml_probe_read(vm_offset_t paddr, unsigned int *val)
{
- if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4)
- return FALSE;
+ if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4) {
+ return FALSE;
+ }
- *val = ml_phys_read((pmap_paddr_t)paddr);
+ *val = ml_phys_read(paddr);
- return TRUE;
+ return TRUE;
}
/*
* have a machine check here. So we have to be able to handle that.
* We assume that machine checks are enabled both in MSR and HIDs
*/
-boolean_t
+boolean_t
ml_probe_read_64(addr64_t paddr64, unsigned int *val)
{
- if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4)
- return FALSE;
+ if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4) {
+ return FALSE;
+ }
- *val = ml_phys_read_64((pmap_paddr_t)paddr64);
- return TRUE;
+ *val = ml_phys_read_64(paddr64);
+ return TRUE;
}
-int bcmp(
- const void *pa,
- const void *pb,
- size_t len)
+#undef bcmp
+int
+bcmp(
+ const void *pa,
+ const void *pb,
+ size_t len)
{
const char *a = (const char *)pa;
const char *b = (const char *)pb;
- if (len == 0)
+ if (len == 0) {
return 0;
+ }
- do
- if (*a++ != *b++)
+ do{
+ if (*a++ != *b++) {
break;
- while (--len);
+ }
+ } while (--len);
return (int)len;
}
+#undef memcmp
int
memcmp(const void *s1, const void *s2, size_t n)
{
const unsigned char *p1 = s1, *p2 = s2;
do {
- if (*p1++ != *p2++)
- return (*--p1 - *--p2);
+ if (*p1++ != *p2++) {
+ return *--p1 - *--p2;
+ }
} while (--n != 0);
}
- return (0);
+ return 0;
+}
+
+#undef memmove
+void *
+memmove(void *dst, const void *src, size_t ulen)
+{
+ bcopy(src, dst, ulen);
+ return dst;
}
/*
* the terminating null character.
*/
+#undef strlen
size_t
strlen(
- register const char *string)
+ const char *string)
{
- register const char *ret = string;
+ const char *ret = string;
- while (*string++ != '\0')
+ while (*string++ != '\0') {
continue;
+ }
return string - 1 - ret;
}
-uint32_t
-hw_compare_and_store(uint32_t oldval, uint32_t newval, volatile uint32_t *dest)
-{
- return OSCompareAndSwap((UInt32)oldval,
- (UInt32)newval,
- (volatile UInt32 *)dest);
-}
-
-#if MACH_ASSERT
+#if MACH_ASSERT
/*
* Machine-dependent routine to fill in an array with up to callstack_max
* levels of return pc information.
*/
-void machine_callstack(
- __unused natural_t *buf,
- __unused vm_size_t callstack_max)
+void
+machine_callstack(
+ __unused uintptr_t *buf,
+ __unused vm_size_t callstack_max)
{
}
-#endif /* MACH_ASSERT */
+#endif /* MACH_ASSERT */
-void fillPage(ppnum_t pa, unsigned int fill)
+void
+fillPage(ppnum_t pa, unsigned int fill)
{
- pmap_paddr_t src;
- int i;
+ uint64_t src;
int cnt = PAGE_SIZE / sizeof(unsigned int);
- unsigned int *addr;
src = i386_ptob(pa);
- for (i = 0, addr = (unsigned int *)PHYSMAP_PTOV(src); i < cnt; i++)
- *addr++ = fill;
+ memset_word((int *)PHYSMAP_PTOV(src), fill, cnt);
}
-static inline void __sfence(void)
-{
- __asm__ volatile("sfence");
-}
-static inline void __mfence(void)
-{
- __asm__ volatile("mfence");
-}
-static inline void __wbinvd(void)
-{
- __asm__ volatile("wbinvd");
-}
-static inline void __clflush(void *ptr)
+static inline void
+__clflush(void *ptr)
{
- __asm__ volatile("clflush (%0)" : : "r" (ptr));
+ __asm__ volatile ("clflush (%0)" : : "r" (ptr));
}
-void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
+void
+dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
{
addr64_t linesize = cpuid_info()->cache_linesize;
addr64_t bound = (pa + count + linesize - 1) & ~(linesize - 1);
- __mfence();
+ mfence();
while (pa < bound) {
__clflush(PHYSMAP_PTOV(pa));
pa += linesize;
}
- __mfence();
+ mfence();
}
-void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)
+void
+dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)
{
- return(dcache_incoherent_io_store64(pa,count));
+ return dcache_incoherent_io_store64(pa, count);
}
void
{
if (phys) {
dcache_incoherent_io_flush64(addr, count);
- }
- else {
- uint32_t linesize = cpuid_info()->cache_linesize;
- addr64_t bound = (addr + count + linesize -1) & ~(linesize - 1);
- __mfence();
+ } else {
+ uint64_t linesize = cpuid_info()->cache_linesize;
+ addr64_t bound = (addr + count + linesize - 1) & ~(linesize - 1);
+ mfence();
while (addr < bound) {
__clflush((void *) (uintptr_t) addr);
addr += linesize;
}
- __mfence();
+ mfence();
}
}
void
invalidate_icache64(__unused addr64_t addr,
- __unused unsigned count,
- __unused int phys)
+ __unused unsigned count,
+ __unused int phys)
{
}
void
mapping_set_mod(ppnum_t pn)
{
- pmap_set_modify(pn);
+ pmap_set_modify(pn);
}
void
mapping_set_ref(ppnum_t pn)
{
- pmap_set_reference(pn);
+ pmap_set_reference(pn);
}
+extern i386_cpu_info_t cpuid_cpu_info;
void
cache_flush_page_phys(ppnum_t pa)
{
- boolean_t istate;
- unsigned char *cacheline_addr;
- int cacheline_size = cpuid_info()->cache_linesize;
- int cachelines_to_flush = PAGE_SIZE/cacheline_size;
+ boolean_t istate;
+ unsigned char *cacheline_addr;
+ i386_cpu_info_t *cpuid_infop = cpuid_info();
+ int cacheline_size;
+ int cachelines_to_flush;
+
+ cacheline_size = cpuid_infop->cache_linesize;
+ if (cacheline_size == 0) {
+ panic("cacheline_size=0 cpuid_infop=%p\n", cpuid_infop);
+ }
+ cachelines_to_flush = PAGE_SIZE / cacheline_size;
- __mfence();
+ mfence();
istate = ml_set_interrupts_enabled(FALSE);
for (cacheline_addr = (unsigned char *)PHYSMAP_PTOV(i386_ptob(pa));
- cachelines_to_flush > 0;
- cachelines_to_flush--, cacheline_addr += cacheline_size) {
+ cachelines_to_flush > 0;
+ cachelines_to_flush--, cacheline_addr += cacheline_size) {
__clflush((void *) cacheline_addr);
}
(void) ml_set_interrupts_enabled(istate);
- __mfence();
+ mfence();
}
#if !MACH_KDP
void
-kdp_register_callout(void)
+kdp_register_callout(kdp_callout_fn_t fn, void *arg)
{
+#pragma unused(fn,arg)
}
#endif
#if !CONFIG_VMX
-int host_vmxon(boolean_t exclusive __unused)
+int
+host_vmxon(boolean_t exclusive __unused)
{
return VMX_UNSUPPORTED;
}
-void host_vmxoff(void)
+void
+host_vmxoff(void)
{
return;
}
projects / apple / xnu.git / blobdiff