xnu-6153.81.5.tar.gz

[apple/xnu.git] / osfmk / x86_64 / loose_ends.c

diff --git a/osfmk/x86_64/loose_ends.c b/osfmk/x86_64/loose_ends.c
index 3d75d8eab4573081aaea2ad4c706d8297e7e3fad..807ecfc52c652b7f0b048e567d16fa36c669015a 100644 (file)
--- a/osfmk/x86_64/loose_ends.c
+++ b/osfmk/x86_64/loose_ends.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2019 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -76,8 +76,20 @@
 #include <vm/vm_fault.h>
 
 #include <libkern/OSAtomic.h>
 #include <vm/vm_fault.h>
 
 #include <libkern/OSAtomic.h>

+#include <libkern/OSDebug.h>

 #include <sys/kdebug.h>
 
 #include <sys/kdebug.h>
 

+#if !MACH_KDP
+#include <kdp/kdp_callout.h>
+#endif /* !MACH_KDP */
+
+#include <architecture/i386/pio.h>
+
+#include <libkern/OSDebug.h>
+#if CONFIG_DTRACE
+#include <mach/sdt.h>
+#endif
+

 #if 0
 
 #undef KERNEL_DEBUG
 #if 0
 
 #undef KERNEL_DEBUG


@@ -86,25 +98,28 @@
 
 #endif
 
 
 #endif
 

+/* prevent infinite recursion when memmove calls bcopy; in string.h, bcopy is defined to call memmove */
+#undef bcopy
+

 /* XXX - should be gone from here */
 /* XXX - should be gone from here */

-extern void            invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
-extern void            flush_dcache64(addr64_t addr, unsigned count, int phys);
-extern boolean_t       phys_page_exists(ppnum_t);
-extern void            bcopy_no_overwrite(const char *from, char *to,vm_size_t bytes);
-extern void            pmap_set_reference(ppnum_t pn);
-extern void            mapping_set_mod(ppnum_t pa); 
-extern void            mapping_set_ref(ppnum_t pn);
+extern void             invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
+extern void             flush_dcache64(addr64_t addr, unsigned count, int phys);
+extern boolean_t        phys_page_exists(ppnum_t);
+extern void             bcopy_no_overwrite(const char *from, char *to, vm_size_t bytes);
+extern void             pmap_set_reference(ppnum_t pn);
+extern void             mapping_set_mod(ppnum_t pa);
+extern void             mapping_set_ref(ppnum_t pn);

 
 

-extern void            ovbcopy(const char      *from,
-                               char            *to,
-                               vm_size_t       nbytes);
-void machine_callstack(natural_t *buf, vm_size_t callstack_max);
+extern void             ovbcopy(const char      *from,
+    char            *to,
+    vm_size_t       nbytes);
+void machine_callstack(uintptr_t *buf, vm_size_t callstack_max);

 
 
 #define value_64bit(value)  ((value) & 0xFFFFFFFF00000000ULL)
 #define low32(x)  ((unsigned int)((x) & 0x00000000FFFFFFFFULL))
 
 
 
 #define value_64bit(value)  ((value) & 0xFFFFFFFF00000000ULL)
 #define low32(x)  ((unsigned int)((x) & 0x00000000FFFFFFFFULL))
 

-#define INT_SIZE       (BYTE_SIZE * sizeof (int))
+#define INT_SIZE        (BYTE_SIZE * sizeof (int))

 
 /*
  * Set indicated bit in bit string.
 
 /*
  * Set indicated bit in bit string.


@@ -141,15 +156,18 @@ ffsbit(int *s)
 {
        int             offset;
 
 {
        int             offset;
 

-       for (offset = 0; !*s; offset += (int)INT_SIZE, ++s);
+       for (offset = 0; !*s; offset += (int)INT_SIZE, ++s) {
+               ;
+       }

        return offset + __builtin_ctz(*s);
 }
 
 int
 ffs(unsigned int mask)
 {
        return offset + __builtin_ctz(*s);
 }
 
 int
 ffs(unsigned int mask)
 {

-       if (mask == 0)
+       if (mask == 0) {

                return 0;
                return 0;

+       }

 
        /*
         * NOTE: cannot use __builtin_ffs because it generates a call to
 
        /*
         * NOTE: cannot use __builtin_ffs because it generates a call to


@@ -158,18 +176,55 @@ ffs(unsigned int mask)
        return 1 + __builtin_ctz(mask);
 }
 
        return 1 + __builtin_ctz(mask);
 }
 

+int
+ffsll(unsigned long long mask)
+{
+       if (mask == 0) {
+               return 0;
+       }
+
+       /*
+        * NOTE: cannot use __builtin_ffsll because it generates a call to
+        * 'ffsll'
+        */
+       return 1 + __builtin_ctzll(mask);
+}
+
+/*
+ * Find last bit set in bit string.
+ */
+int
+fls(unsigned int mask)
+{
+       if (mask == 0) {
+               return 0;
+       }
+
+       return (sizeof(mask) << 3) - __builtin_clz(mask);
+}
+
+int
+flsll(unsigned long long mask)
+{
+       if (mask == 0) {
+               return 0;
+       }
+
+       return (sizeof(mask) << 3) - __builtin_clzll(mask);
+}
+

 void
 bzero_phys_nc(
 void
 bzero_phys_nc(

-             addr64_t src64,
-             uint32_t bytes)
+       addr64_t src64,
+       uint32_t bytes)

 {
 {

-  bzero_phys(src64,bytes);
+       bzero_phys(src64, bytes);

 }
 
 void
 bzero_phys(
 }
 
 void
 bzero_phys(

-          addr64_t src64,
-          uint32_t bytes)
+       addr64_t src64,
+       uint32_t bytes)

 {
        bzero(PHYSMAP_PTOV(src64), bytes);
 }
 {
        bzero(PHYSMAP_PTOV(src64), bytes);
 }


@@ -181,14 +236,14 @@ bzero_phys(
 
 void
 bcopy_phys(
 
 void
 bcopy_phys(

-          addr64_t src64,
-          addr64_t dst64,
-          vm_size_t bytes)
+       addr64_t src64,
+       addr64_t dst64,
+       vm_size_t bytes)

 {
        /* Not necessary for K64 - but ensure we stay within a page */
 {
        /* Not necessary for K64 - but ensure we stay within a page */

-       if (((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) ||
-            ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
-               panic("bcopy_phys alignment");
+       if (((((uint32_t)src64 & (NBPG - 1)) + bytes) > NBPG) ||
+           ((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) {
+               panic("bcopy_phys alignment");

        }
        bcopy(PHYSMAP_PTOV(src64), PHYSMAP_PTOV(dst64), bytes);
 }
        }
        bcopy(PHYSMAP_PTOV(src64), PHYSMAP_PTOV(dst64), bytes);
 }


@@ -199,41 +254,42 @@ bcopy_phys(
 
 int
 apply_func_phys(
 
 int
 apply_func_phys(

-               addr64_t dst64,
-               vm_size_t bytes,
-               int (*func)(void * buffer, vm_size_t bytes, void * arg),
-               void * arg)
+       addr64_t dst64,
+       vm_size_t bytes,
+       int (*func)(void * buffer, vm_size_t bytes, void * arg),
+       void * arg)

 {
        /* Not necessary for K64 - but ensure we stay within a page */
 {
        /* Not necessary for K64 - but ensure we stay within a page */

-       if (((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
-               panic("apply_func_phys alignment");
+       if (((((uint32_t)dst64 & (NBPG - 1)) + bytes) > NBPG)) {
+               panic("apply_func_phys alignment");

        }
 
        return func(PHYSMAP_PTOV(dst64), bytes, arg);
 }
 
        }
 
        return func(PHYSMAP_PTOV(dst64), bytes, arg);
 }
 

-/* 
- * ovbcopy - like bcopy, but recognizes overlapping ranges and handles 
+/*
+ * ovbcopy - like bcopy, but recognizes overlapping ranges and handles

  *           them correctly.
  */
 
 void
 ovbcopy(
  *           them correctly.
  */
 
 void
 ovbcopy(

-       const char      *from,
-       char            *to,
-       vm_size_t       bytes)          /* num bytes to copy */
+       const char      *from,
+       char            *to,
+       vm_size_t       bytes)          /* num bytes to copy */

 {
        /* Assume that bcopy copies left-to-right (low addr first). */
 {
        /* Assume that bcopy copies left-to-right (low addr first). */

-       if (from + bytes <= to || to + bytes <= from || to == from)
-               bcopy_no_overwrite(from, to, bytes);    /* non-overlapping or no-op*/
-       else if (from > to)
-               bcopy_no_overwrite(from, to, bytes);    /* overlapping but OK */
-       else {
+       if (from + bytes <= to || to + bytes <= from || to == from) {
+               bcopy_no_overwrite(from, to, bytes);    /* non-overlapping or no-op*/
+       } else if (from > to) {
+               bcopy_no_overwrite(from, to, bytes);    /* overlapping but OK */
+       } else {

                /* to > from: overlapping, and must copy right-to-left. */
                from += bytes - 1;
                to += bytes - 1;
                /* to > from: overlapping, and must copy right-to-left. */
                from += bytes - 1;
                to += bytes - 1;

-               while (bytes-- > 0)
+               while (bytes-- > 0) {

                        *to-- = *from--;
                        *to-- = *from--;

+               }

        }
 }
 
        }
 }
 


@@ -242,86 +298,171 @@ ovbcopy(
  *  Read data from a physical address. Memory should not be cache inhibited.
  */
 
  *  Read data from a physical address. Memory should not be cache inhibited.
  */
 

+uint64_t reportphyreaddelayabs;
+uint64_t reportphywritedelayabs;
+uint32_t reportphyreadosbt;
+uint32_t reportphywriteosbt;
+
+#if DEVELOPMENT || DEBUG
+uint32_t phyreadpanic = 1;
+uint32_t phywritepanic = 1;
+uint64_t tracephyreaddelayabs = 50 * NSEC_PER_USEC;
+uint64_t tracephywritedelayabs = 50 * NSEC_PER_USEC;
+uint64_t simulate_stretched_io = 0;
+#else
+uint32_t phyreadpanic = 0;
+uint32_t phywritepanic = 0;
+uint64_t tracephyreaddelayabs = 0;
+uint64_t tracephywritedelayabs = 0;
+#endif

 
 

-static inline unsigned int
-ml_phys_read_data(pmap_paddr_t paddr, int size)
+__private_extern__ uint64_t
+ml_phys_read_data(uint64_t paddr, int size)

 {
 {

-       unsigned int result;
+       uint64_t result = 0;
+       unsigned char s1;
+       unsigned short s2;
+       boolean_t istate = TRUE, timeread = FALSE;
+       uint64_t sabs = 0, eabs;
+
+       if (__improbable(!physmap_enclosed(paddr))) {
+               panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
+       }
+
+       if (__improbable(reportphyreaddelayabs != 0)) {
+               istate = ml_set_interrupts_enabled(FALSE);
+               sabs = mach_absolute_time();
+               timeread = TRUE;
+       }
+#if DEVELOPMENT || DEBUG
+       if (__improbable(timeread && simulate_stretched_io)) {
+               sabs -= simulate_stretched_io;
+       }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+       switch (size) {
+       case 1:
+               s1 = *(volatile unsigned char *)PHYSMAP_PTOV(paddr);
+               result = s1;
+               break;
+       case 2:
+               s2 = *(volatile unsigned short *)PHYSMAP_PTOV(paddr);
+               result = s2;
+               break;
+       case 4:
+               result = *(volatile unsigned int *)PHYSMAP_PTOV(paddr);
+               break;
+       case 8:
+               result = *(volatile unsigned long long *)PHYSMAP_PTOV(paddr);
+               break;
+       default:
+               panic("Invalid size %d for ml_phys_read_data", size);
+               break;
+       }

 
 

-        switch (size) {
-            unsigned char s1;
-            unsigned short s2;
-        case 1:
-            s1 = *(unsigned char *)PHYSMAP_PTOV(paddr);
-            result = s1;
-            break;
-        case 2:
-            s2 = *(unsigned short *)PHYSMAP_PTOV(paddr);
-            result = s2;
-            break;
-        case 4:
-        default:
-            result = *(unsigned int *)PHYSMAP_PTOV(paddr);
-            break;
-        }
+       if (__improbable(timeread == TRUE)) {
+               eabs = mach_absolute_time();

 
 

-        return result;
+#if DEVELOPMENT || DEBUG
+               iotrace(IOTRACE_PHYS_READ, 0, paddr, size, result, sabs, eabs - sabs);
+#endif
+
+               if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
+                       (void)ml_set_interrupts_enabled(istate);
+
+                       if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
+                               panic_io_port_read();
+                               panic("Read from physical addr 0x%llx took %llu ns, "
+                                   "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
+                                   paddr, (eabs - sabs), result, sabs, eabs,
+                                   reportphyreaddelayabs);
+                       }
+
+                       if (reportphyreadosbt) {
+                               OSReportWithBacktrace("ml_phys_read_data took %lluus",
+                                   (eabs - sabs) / NSEC_PER_USEC);
+                       }
+#if CONFIG_DTRACE
+                       DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs),
+                           uint64_t, paddr, uint32_t, size, uint64_t, result);
+#endif /* CONFIG_DTRACE */
+               } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) {
+                       KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
+                           (eabs - sabs), sabs, paddr, result);
+
+                       (void)ml_set_interrupts_enabled(istate);
+               } else {
+                       (void)ml_set_interrupts_enabled(istate);
+               }
+       }
+
+       return result;

 }
 
 static unsigned long long
 }
 
 static unsigned long long

-ml_phys_read_long_long(pmap_paddr_t paddr )
+ml_phys_read_long_long(uint64_t paddr)

 {
 {

-       return *(unsigned long long *)PHYSMAP_PTOV(paddr);
+       return ml_phys_read_data(paddr, 8);

 }
 
 }
 

-unsigned int ml_phys_read( vm_offset_t paddr)
+unsigned int
+ml_phys_read(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+       return (unsigned int) ml_phys_read_data(paddr, 4);

 }
 
 }
 

-unsigned int ml_phys_read_word(vm_offset_t paddr) {
-
-        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
+unsigned int
+ml_phys_read_word(vm_offset_t paddr)
+{
+       return (unsigned int) ml_phys_read_data(paddr, 4);

 }
 
 }
 

-unsigned int ml_phys_read_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+       return (unsigned int) ml_phys_read_data(paddr64, 4);

 }
 
 }
 

-unsigned int ml_phys_read_word_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_word_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
+       return (unsigned int) ml_phys_read_data(paddr64, 4);

 }
 
 }
 

-unsigned int ml_phys_read_half(vm_offset_t paddr)
+unsigned int
+ml_phys_read_half(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 2);
+       return (unsigned int) ml_phys_read_data(paddr, 2);

 }
 
 }
 

-unsigned int ml_phys_read_half_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_half_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
+       return (unsigned int) ml_phys_read_data(paddr64, 2);

 }
 
 }
 

-unsigned int ml_phys_read_byte(vm_offset_t paddr)
+unsigned int
+ml_phys_read_byte(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr, 1);
+       return (unsigned int) ml_phys_read_data(paddr, 1);

 }
 
 }
 

-unsigned int ml_phys_read_byte_64(addr64_t paddr64)
+unsigned int
+ml_phys_read_byte_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
+       return (unsigned int) ml_phys_read_data(paddr64, 1);

 }
 
 }
 

-unsigned long long ml_phys_read_double(vm_offset_t paddr)
+unsigned long long
+ml_phys_read_double(vm_offset_t paddr)

 {
 {

-        return ml_phys_read_long_long((pmap_paddr_t)paddr);
+       return ml_phys_read_long_long(paddr);

 }
 
 }
 

-unsigned long long ml_phys_read_double_64(addr64_t paddr64)
+unsigned long long
+ml_phys_read_double_64(addr64_t paddr64)

 {
 {

-        return ml_phys_read_long_long((pmap_paddr_t)paddr64);
+       return ml_phys_read_long_long(paddr64);

 }
 
 
 }
 
 


@@ -330,98 +471,342 @@ unsigned long long ml_phys_read_double_64(addr64_t paddr64)
  *  Write data to a physical address. Memory should not be cache inhibited.
  */
 
  *  Write data to a physical address. Memory should not be cache inhibited.
  */
 

-static inline void
-ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
+__private_extern__ void
+ml_phys_write_data(uint64_t paddr, unsigned long long data, int size)
+{
+       boolean_t istate = TRUE, timewrite = FALSE;
+       uint64_t sabs = 0, eabs;
+
+       if (__improbable(!physmap_enclosed(paddr))) {
+               panic("%s: 0x%llx out of bounds\n", __FUNCTION__, paddr);
+       }
+
+       if (__improbable(reportphywritedelayabs != 0)) {
+               istate = ml_set_interrupts_enabled(FALSE);
+               sabs = mach_absolute_time();
+               timewrite = TRUE;
+       }
+#if DEVELOPMENT || DEBUG
+       if (__improbable(timewrite && simulate_stretched_io)) {
+               sabs -= simulate_stretched_io;
+       }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+       switch (size) {
+       case 1:
+               *(volatile unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data;
+               break;
+       case 2:
+               *(volatile unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data;
+               break;
+       case 4:
+               *(volatile unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data;
+               break;
+       case 8:
+               *(volatile unsigned long *)PHYSMAP_PTOV(paddr) = data;
+               break;
+       default:
+               panic("Invalid size %d for ml_phys_write_data", size);
+               break;
+       }
+
+       if (__improbable(timewrite == TRUE)) {
+               eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+               iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs);
+#endif
+
+               if (__improbable((eabs - sabs) > reportphywritedelayabs)) {
+                       (void)ml_set_interrupts_enabled(istate);
+
+                       if (phywritepanic && (machine_timeout_suspended() == FALSE)) {
+                               panic_io_port_read();
+                               panic("Write to physical addr 0x%llx took %llu ns, "
+                                   "data: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
+                                   paddr, (eabs - sabs), data, sabs, eabs,
+                                   reportphywritedelayabs);
+                       }
+
+                       if (reportphywriteosbt) {
+                               OSReportWithBacktrace("ml_phys_write_data (%p, 0x%llx) "
+                                   "took %lluus",
+                                   paddr, data, (eabs - sabs) / NSEC_PER_USEC);
+                       }
+#if CONFIG_DTRACE
+                       DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs),
+                           uint64_t, paddr, uint32_t, size, uint64_t, data);
+#endif /* CONFIG_DTRACE */
+               } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) {
+                       KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
+                           (eabs - sabs), sabs, paddr, data);
+
+                       (void)ml_set_interrupts_enabled(istate);
+               } else {
+                       (void)ml_set_interrupts_enabled(istate);
+               }
+       }
+}
+
+void
+ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
+{
+       ml_phys_write_data(paddr, data, 1);
+}
+
+void
+ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
+{
+       ml_phys_write_data(paddr64, data, 1);
+}
+
+void
+ml_phys_write_half(vm_offset_t paddr, unsigned int data)

 {
 {

-        switch (size) {
-        case 1:
-           *(unsigned char *)PHYSMAP_PTOV(paddr) = (unsigned char)data;
-            break;
-        case 2:
-           *(unsigned short *)PHYSMAP_PTOV(paddr) = (unsigned short)data;
-            break;
-        case 4:
-        default:
-           *(unsigned int *)PHYSMAP_PTOV(paddr) = (unsigned int)data;
-            break;
-        }
+       ml_phys_write_data(paddr, data, 2);

 }
 
 }
 

-static void
-ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
+void
+ml_phys_write_half_64(addr64_t paddr64, unsigned int data)

 {
 {

-       *(unsigned long long *)PHYSMAP_PTOV(paddr) = data;
+       ml_phys_write_data(paddr64, data, 2);

 }
 
 }
 

-void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
+       ml_phys_write_data(paddr, data, 4);

 }
 
 }
 

-void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
+       ml_phys_write_data(paddr64, data, 4);

 }
 
 }
 

-void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_word(vm_offset_t paddr, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
+       ml_phys_write_data(paddr, data, 4);

 }
 
 }
 

-void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_word_64(addr64_t paddr64, unsigned int data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
+       ml_phys_write_data(paddr64, data, 4);

 }
 
 }
 

-void ml_phys_write(vm_offset_t paddr, unsigned int data)
+void
+ml_phys_write_double(vm_offset_t paddr, unsigned long long data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+       ml_phys_write_data(paddr, data, 8);

 }
 
 }
 

-void ml_phys_write_64(addr64_t paddr64, unsigned int data)
+void
+ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+       ml_phys_write_data(paddr64, data, 8);

 }
 
 }
 

-void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
+uint32_t
+ml_port_io_read(uint16_t ioport, int size)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
+       uint32_t result = 0;
+
+       uint64_t sabs, eabs;
+       boolean_t istate, timeread = FALSE;
+
+       if (__improbable(reportphyreaddelayabs != 0)) {
+               istate = ml_set_interrupts_enabled(FALSE);
+               sabs = mach_absolute_time();
+               timeread = TRUE;
+       }
+
+#if DEVELOPMENT || DEBUG
+       if (__improbable(timeread && simulate_stretched_io)) {
+               sabs -= simulate_stretched_io;
+       }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+       switch (size) {
+       case 1:
+               result = inb(ioport);
+               break;
+       case 2:
+               result = inw(ioport);
+               break;
+       case 4:
+               result = inl(ioport);
+               break;
+       default:
+               panic("Invalid size %d for ml_port_io_read(0x%x)", size, (unsigned)ioport);
+               break;
+       }
+
+       if (__improbable(timeread == TRUE)) {
+               eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+               iotrace(IOTRACE_PORTIO_READ, 0, ioport, size, result, sabs, eabs - sabs);
+#endif
+
+               if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
+                       (void)ml_set_interrupts_enabled(istate);
+
+                       if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
+                               panic_io_port_read();
+                               panic("Read from IO port 0x%x took %llu ns, "
+                                   "result: 0x%x (start: %llu, end: %llu), ceiling: %llu",
+                                   ioport, (eabs - sabs), result, sabs, eabs,
+                                   reportphyreaddelayabs);
+                       }
+
+                       if (reportphyreadosbt) {
+                               OSReportWithBacktrace("ml_port_io_read(0x%x) took %lluus",
+                                   ioport, (eabs - sabs) / NSEC_PER_USEC);
+                       }
+#if CONFIG_DTRACE
+                       DTRACE_PHYSLAT3(portioread, uint64_t, (eabs - sabs),
+                           uint16_t, ioport, uint32_t, size);
+#endif /* CONFIG_DTRACE */
+               } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) {
+                       KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_READ),
+                           (eabs - sabs), sabs, ioport, result);
+
+                       (void)ml_set_interrupts_enabled(istate);
+               } else {
+                       (void)ml_set_interrupts_enabled(istate);
+               }
+       }
+
+       return result;
+}
+
+void
+ml_port_io_write(uint16_t ioport, uint32_t val, int size)
+{
+       uint64_t sabs, eabs;
+       boolean_t istate, timewrite = FALSE;
+
+       if (__improbable(reportphywritedelayabs != 0)) {
+               istate = ml_set_interrupts_enabled(FALSE);
+               sabs = mach_absolute_time();
+               timewrite = TRUE;
+       }
+#if DEVELOPMENT || DEBUG
+       if (__improbable(timewrite && simulate_stretched_io)) {
+               sabs -= simulate_stretched_io;
+       }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+
+       switch (size) {
+       case 1:
+               outb(ioport, (uint8_t)val);
+               break;
+       case 2:
+               outw(ioport, (uint16_t)val);
+               break;
+       case 4:
+               outl(ioport, (uint32_t)val);
+               break;
+       default:
+               panic("Invalid size %d for ml_port_io_write(0x%x)", size, (unsigned)ioport);
+               break;
+       }
+
+       if (__improbable(timewrite == TRUE)) {
+               eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+               iotrace(IOTRACE_PORTIO_WRITE, 0, ioport, size, val, sabs, eabs - sabs);
+#endif
+
+               if (__improbable((eabs - sabs) > reportphywritedelayabs)) {
+                       (void)ml_set_interrupts_enabled(istate);
+
+                       if (phywritepanic && (machine_timeout_suspended() == FALSE)) {
+                               panic_io_port_read();
+                               panic("Write to IO port 0x%x took %llu ns, val: 0x%x"
+                                   " (start: %llu, end: %llu), ceiling: %llu",
+                                   ioport, (eabs - sabs), val, sabs, eabs,
+                                   reportphywritedelayabs);
+                       }
+
+                       if (reportphywriteosbt) {
+                               OSReportWithBacktrace("ml_port_io_write(0x%x, %d, 0x%llx) "
+                                   "took %lluus",
+                                   ioport, size, val, (eabs - sabs) / NSEC_PER_USEC);
+                       }
+
+#if CONFIG_DTRACE
+                       DTRACE_PHYSLAT4(portiowrite, uint64_t, (eabs - sabs),
+                           uint16_t, ioport, uint32_t, size, uint64_t, val);
+#endif /* CONFIG_DTRACE */
+               } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) {
+                       KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PORTIO_WRITE),
+                           (eabs - sabs), sabs, ioport, val);
+
+                       (void)ml_set_interrupts_enabled(istate);
+               } else {
+                       (void)ml_set_interrupts_enabled(istate);
+               }
+       }

 }
 
 }
 

-void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
+uint8_t
+ml_port_io_read8(uint16_t ioport)

 {
 {

-        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
+       return ml_port_io_read(ioport, 1);

 }
 
 }
 

-void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
+uint16_t
+ml_port_io_read16(uint16_t ioport)

 {
 {

-        ml_phys_write_long_long((pmap_paddr_t)paddr, data);
+       return ml_port_io_read(ioport, 2);

 }
 
 }
 

-void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
+uint32_t
+ml_port_io_read32(uint16_t ioport)

 {
 {

-        ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
+       return ml_port_io_read(ioport, 4);

 }
 
 }
 

+void
+ml_port_io_write8(uint16_t ioport, uint8_t val)
+{
+       ml_port_io_write(ioport, val, 1);
+}
+
+void
+ml_port_io_write16(uint16_t ioport, uint16_t val)
+{
+       ml_port_io_write(ioport, val, 2);
+}
+
+void
+ml_port_io_write32(uint16_t ioport, uint32_t val)
+{
+       ml_port_io_write(ioport, val, 4);
+}

 
 /* PCI config cycle probing
  *
  *
  *      Read the memory location at physical address paddr.
 
 /* PCI config cycle probing
  *
  *
  *      Read the memory location at physical address paddr.

- *  This is a part of a device probe, so there is a good chance we will
- *  have a machine check here. So we have to be able to handle that.
- *  We assume that machine checks are enabled both in MSR and HIDs
+ * *Does not* recover from machine checks, unlike the PowerPC implementation.
+ * Should probably be deprecated.

  */
 
 boolean_t
 ml_probe_read(vm_offset_t paddr, unsigned int *val)
 {
  */
 
 boolean_t
 ml_probe_read(vm_offset_t paddr, unsigned int *val)
 {

-    if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4)
-        return FALSE;
+       if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4) {
+               return FALSE;
+       }

 
 

-    *val = ml_phys_read((pmap_paddr_t)paddr);
+       *val = ml_phys_read(paddr);

 
 

-    return TRUE;
+       return TRUE;

 }
 
 /*
 }
 
 /*


@@ -430,36 +815,42 @@ ml_probe_read(vm_offset_t paddr, unsigned int *val)
  *  have a machine check here. So we have to be able to handle that.
  *  We assume that machine checks are enabled both in MSR and HIDs
  */
  *  have a machine check here. So we have to be able to handle that.
  *  We assume that machine checks are enabled both in MSR and HIDs
  */

-boolean_t 
+boolean_t

 ml_probe_read_64(addr64_t paddr64, unsigned int *val)
 {
 ml_probe_read_64(addr64_t paddr64, unsigned int *val)
 {

-    if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4)
-        return FALSE;
+       if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4) {
+               return FALSE;
+       }

 
 

-    *val = ml_phys_read_64((pmap_paddr_t)paddr64);
-    return TRUE;
+       *val = ml_phys_read_64(paddr64);
+       return TRUE;

 }
 
 
 }
 
 

-int bcmp(
-       const void      *pa,
-       const void      *pb,
-       size_t  len)
+#undef bcmp
+int
+bcmp(
+       const void      *pa,
+       const void      *pb,
+       size_t  len)

 {
        const char *a = (const char *)pa;
        const char *b = (const char *)pb;
 
 {
        const char *a = (const char *)pa;
        const char *b = (const char *)pb;
 

-       if (len == 0)
+       if (len == 0) {

                return 0;
                return 0;

+       }

 
 

-       do
-               if (*a++ != *b++)
+       do{
+               if (*a++ != *b++) {

                        break;
                        break;

-       while (--len);
+               }
+       } while (--len);

 
        return (int)len;
 }
 
 
        return (int)len;
 }
 

+#undef memcmp

 int
 memcmp(const void *s1, const void *s2, size_t n)
 {
 int
 memcmp(const void *s1, const void *s2, size_t n)
 {


@@ -467,11 +858,20 @@ memcmp(const void *s1, const void *s2, size_t n)
                const unsigned char *p1 = s1, *p2 = s2;
 
                do {
                const unsigned char *p1 = s1, *p2 = s2;
 
                do {

-                       if (*p1++ != *p2++)
-                               return (*--p1 - *--p2);
+                       if (*p1++ != *p2++) {
+                               return *--p1 - *--p2;
+                       }

                } while (--n != 0);
        }
                } while (--n != 0);
        }

-       return (0);
+       return 0;
+}
+
+#undef memmove
+void *
+memmove(void *dst, const void *src, size_t ulen)
+{
+       bcopy(src, dst, ulen);
+       return dst;

 }
 
 /*
 }
 
 /*


@@ -480,86 +880,70 @@ memcmp(const void *s1, const void *s2, size_t n)
  * the terminating null character.
  */
 
  * the terminating null character.
  */
 

+#undef strlen

 size_t
 strlen(
 size_t
 strlen(

-       register const char *string)
+       const char *string)

 {
 {

-       register const char *ret = string;
+       const char *ret = string;

 
 

-       while (*string++ != '\0')
+       while (*string++ != '\0') {

                continue;
                continue;

+       }

        return string - 1 - ret;
 }
 
        return string - 1 - ret;
 }
 

-uint32_t
-hw_compare_and_store(uint32_t oldval, uint32_t newval, volatile uint32_t *dest)
-{
-       return OSCompareAndSwap((UInt32)oldval,
-                               (UInt32)newval,
-                               (volatile UInt32 *)dest);
-}
-
-#if    MACH_ASSERT
+#if     MACH_ASSERT

 
 /*
  * Machine-dependent routine to fill in an array with up to callstack_max
  * levels of return pc information.
  */
 
 /*
  * Machine-dependent routine to fill in an array with up to callstack_max
  * levels of return pc information.
  */

-void machine_callstack(
-       __unused natural_t      *buf,
-       __unused vm_size_t      callstack_max)
+void
+machine_callstack(
+       __unused uintptr_t      *buf,
+       __unused vm_size_t      callstack_max)

 {
 }
 
 {
 }
 

-#endif /* MACH_ASSERT */
+#endif  /* MACH_ASSERT */

 
 

-void fillPage(ppnum_t pa, unsigned int fill)
+void
+fillPage(ppnum_t pa, unsigned int fill)

 {
 {

-       pmap_paddr_t    src;
-       int             i;
+       uint64_t        src;

        int             cnt = PAGE_SIZE / sizeof(unsigned int);
        int             cnt = PAGE_SIZE / sizeof(unsigned int);

-       unsigned int   *addr;

 
        src = i386_ptob(pa);
 
        src = i386_ptob(pa);

-       for (i = 0, addr = (unsigned int *)PHYSMAP_PTOV(src); i < cnt; i++)
-               *addr++ = fill;
+       memset_word((int *)PHYSMAP_PTOV(src), fill, cnt);

 }
 
 }
 

-static inline void __sfence(void)
-{
-    __asm__ volatile("sfence");
-}
-static inline void __mfence(void)
-{
-    __asm__ volatile("mfence");
-}
-static inline void __wbinvd(void)
-{
-    __asm__ volatile("wbinvd");
-}
-static inline void __clflush(void *ptr)
+static inline void
+__clflush(void *ptr)

 {
 {

-       __asm__ volatile("clflush (%0)" : : "r" (ptr));
+       __asm__ volatile ("clflush (%0)" : : "r" (ptr));

 }
 
 }
 

-void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
+void
+dcache_incoherent_io_store64(addr64_t pa, unsigned int count)

 {
        addr64_t  linesize = cpuid_info()->cache_linesize;
        addr64_t  bound = (pa + count + linesize - 1) & ~(linesize - 1);
 
 {
        addr64_t  linesize = cpuid_info()->cache_linesize;
        addr64_t  bound = (pa + count + linesize - 1) & ~(linesize - 1);
 

-       __mfence();
+       mfence();

 
        while (pa < bound) {
                __clflush(PHYSMAP_PTOV(pa));
                pa += linesize;
        }
 
 
        while (pa < bound) {
                __clflush(PHYSMAP_PTOV(pa));
                pa += linesize;
        }
 

-       __mfence();
+       mfence();

 }
 
 }
 

-void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)
+void
+dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)

 {
 {

-    return(dcache_incoherent_io_store64(pa,count));
+       return dcache_incoherent_io_store64(pa, count);

 }
 
 void
 }
 
 void


@@ -567,23 +951,22 @@ flush_dcache64(addr64_t addr, unsigned count, int phys)
 {
        if (phys) {
                dcache_incoherent_io_flush64(addr, count);
 {
        if (phys) {
                dcache_incoherent_io_flush64(addr, count);

-       }
-       else {
-               uint32_t  linesize = cpuid_info()->cache_linesize;
-               addr64_t  bound = (addr + count + linesize -1) & ~(linesize - 1);
-               __mfence();
+       } else {
+               uint64_t  linesize = cpuid_info()->cache_linesize;
+               addr64_t  bound = (addr + count + linesize - 1) & ~(linesize - 1);
+               mfence();

                while (addr < bound) {
                        __clflush((void *) (uintptr_t) addr);
                        addr += linesize;
                }
                while (addr < bound) {
                        __clflush((void *) (uintptr_t) addr);
                        addr += linesize;
                }

-               __mfence();
+               mfence();

        }
 }
 
 void
 invalidate_icache64(__unused addr64_t addr,
        }
 }
 
 void
 invalidate_icache64(__unused addr64_t addr,

-                   __unused unsigned count,
-                   __unused int phys)
+    __unused unsigned count,
+    __unused int phys)

 {
 }
 
 {
 }
 


@@ -593,53 +976,64 @@ addr64_t         vm_last_addr;
 void
 mapping_set_mod(ppnum_t pn)
 {
 void
 mapping_set_mod(ppnum_t pn)
 {

-  pmap_set_modify(pn);
+       pmap_set_modify(pn);

 }
 
 void
 mapping_set_ref(ppnum_t pn)
 {
 }
 
 void
 mapping_set_ref(ppnum_t pn)
 {

-  pmap_set_reference(pn);
+       pmap_set_reference(pn);

 }
 
 }
 

+extern i386_cpu_info_t  cpuid_cpu_info;

 void
 cache_flush_page_phys(ppnum_t pa)
 {
 void
 cache_flush_page_phys(ppnum_t pa)
 {

-       boolean_t       istate;
-       unsigned char   *cacheline_addr;
-       int             cacheline_size = cpuid_info()->cache_linesize;
-       int             cachelines_to_flush = PAGE_SIZE/cacheline_size;
+       boolean_t       istate;
+       unsigned char   *cacheline_addr;
+       i386_cpu_info_t *cpuid_infop = cpuid_info();
+       int             cacheline_size;
+       int             cachelines_to_flush;
+
+       cacheline_size = cpuid_infop->cache_linesize;
+       if (cacheline_size == 0) {
+               panic("cacheline_size=0 cpuid_infop=%p\n", cpuid_infop);
+       }
+       cachelines_to_flush = PAGE_SIZE / cacheline_size;

 
 

-       __mfence();
+       mfence();

 
        istate = ml_set_interrupts_enabled(FALSE);
 
        for (cacheline_addr = (unsigned char *)PHYSMAP_PTOV(i386_ptob(pa));
 
        istate = ml_set_interrupts_enabled(FALSE);
 
        for (cacheline_addr = (unsigned char *)PHYSMAP_PTOV(i386_ptob(pa));

-            cachelines_to_flush > 0;
-            cachelines_to_flush--, cacheline_addr += cacheline_size) {
+           cachelines_to_flush > 0;
+           cachelines_to_flush--, cacheline_addr += cacheline_size) {

                __clflush((void *) cacheline_addr);
        }
 
        (void) ml_set_interrupts_enabled(istate);
 
                __clflush((void *) cacheline_addr);
        }
 
        (void) ml_set_interrupts_enabled(istate);
 

-       __mfence();
+       mfence();

 }
 
 
 #if !MACH_KDP
 void
 }
 
 
 #if !MACH_KDP
 void

-kdp_register_callout(void)
+kdp_register_callout(kdp_callout_fn_t fn, void *arg)

 {
 {

+#pragma unused(fn,arg)

 }
 #endif
 
 #if !CONFIG_VMX
 }
 #endif
 
 #if !CONFIG_VMX

-int host_vmxon(boolean_t exclusive __unused)
+int
+host_vmxon(boolean_t exclusive __unused)

 {
        return VMX_UNSUPPORTED;
 }
 
 {
        return VMX_UNSUPPORTED;
 }
 

-void host_vmxoff(void)
+void
+host_vmxoff(void)

 {
        return;
 }
 {
        return;
 }