xnu-4570.41.2.tar.gz

[apple/xnu.git] / osfmk / i386 / pmap.h

diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h
index 939e47174605d98a377a3a33963f02104a7b63e4..3458ce7cf57806b6f60624d5275fc5944582e3fe 100644 (file)
--- a/osfmk/i386/pmap.h
+++ b/osfmk/i386/pmap.h
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -70,7 +70,6 @@
 
 #ifndef        ASSEMBLER
 
 
 #ifndef        ASSEMBLER
 

-

 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>
 #include <mach/vm_prot.h>
 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>
 #include <mach/vm_prot.h>


@@ -96,9 +95,6 @@
 #define        intel_ptob(x)           i386_ptob(x)
 #define        intel_round_page(x)     i386_round_page(x)
 #define        intel_trunc_page(x)     i386_trunc_page(x)
 #define        intel_ptob(x)           i386_ptob(x)
 #define        intel_round_page(x)     i386_round_page(x)
 #define        intel_trunc_page(x)     i386_trunc_page(x)

-#define trunc_intel_to_vm(x)   trunc_i386_to_vm(x)
-#define round_intel_to_vm(x)   round_i386_to_vm(x)
-#define vm_to_intel(x)         vm_to_i386(x)

 
 /*
  *     i386/i486/i860 Page Table Entry
 
 /*
  *     i386/i486/i860 Page Table Entry


@@ -113,10 +109,7 @@
 
 #define PTESHIFT        12ULL
 
 
 #define PTESHIFT        12ULL
 

-
-#ifdef __x86_64__

 #define LOW_4GB_MASK   ((vm_offset_t)0x00000000FFFFFFFFUL)
 #define LOW_4GB_MASK   ((vm_offset_t)0x00000000FFFFFFFFUL)

-#endif

 
 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */
 
 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */


@@ -170,17 +163,25 @@ typedef uint64_t        pt_entry_t;
 
 typedef uint64_t  pmap_paddr_t;
 
 
 typedef uint64_t  pmap_paddr_t;
 

-#if    DEBUG
+#if    DEVELOPMENT || DEBUG

 #define PMAP_ASSERT 1
 #define PMAP_ASSERT 1

+extern int pmap_asserts_enabled;
+extern int pmap_asserts_traced;

 #endif
 #endif

+

 #if PMAP_ASSERT
 #if PMAP_ASSERT

-#define        pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+#define        pmap_assert(ex) (pmap_asserts_enabled ? ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) : (void)0)

 
 #define pmap_assert2(ex, fmt, args...)                                 \
        do {                                                            \
 
 #define pmap_assert2(ex, fmt, args...)                                 \
        do {                                                            \

-               if (!(ex)) {                                            \
-                       kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args); \
-                       panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args);           \
+               if (__improbable(pmap_asserts_enabled && !(ex))) {      \
+                       if (pmap_asserts_traced) {                      \
+                               KERNEL_DEBUG_CONSTANT(0xDEAD1000, __builtin_return_address(0), __LINE__, 0, 0, 0); \
+                               kdebug_enable = 0;                      \
+                       } else {                                        \
+                                       kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args); \
+                                       panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args); \
+                       }                                               \

                }                                                       \
        } while(0)
 #else
                }                                                       \
        } while(0)
 #else


@@ -189,11 +190,7 @@ typedef uint64_t  pmap_paddr_t;
 #endif
 
 /* superpages */
 #endif
 
 /* superpages */

-#ifdef __x86_64__

 #define SUPERPAGE_NBASEPAGES 512
 #define SUPERPAGE_NBASEPAGES 512

-#else
-#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
-#endif

 
 /*
  * Atomic 64-bit store of a page table entry.
 
 /*
  * Atomic 64-bit store of a page table entry.


@@ -215,8 +212,11 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 
 #define KERNEL_PML4_INDEX              511
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 
 #define KERNEL_PML4_INDEX              511

-#define KERNEL_KEXTS_INDEX     510     /* Home of KEXTs - the basement */
+#define KERNEL_KEXTS_INDEX             510     /* Home of KEXTs - the basement */

 #define KERNEL_PHYSMAP_PML4_INDEX      509     /* virtual to physical map */ 
 #define KERNEL_PHYSMAP_PML4_INDEX      509     /* virtual to physical map */ 

+#define KERNEL_KASAN_PML4_INDEX0       508
+#define KERNEL_KASAN_PML4_INDEX1       507
+#define KERNEL_DBLMAP_PML4_INDEX       (506)

 #define KERNEL_BASE            (0ULL - NBPML4)
 #define KERNEL_BASEMENT                (KERNEL_BASE - NBPML4)
 
 #define KERNEL_BASE            (0ULL - NBPML4)
 #define KERNEL_BASEMENT                (KERNEL_BASE - NBPML4)
 


@@ -226,6 +226,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 /* ?? intel ?? */
 #define VM_WIMG_IO             (VM_MEM_COHERENT |      \
                                VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
 /* ?? intel ?? */
 #define VM_WIMG_IO             (VM_MEM_COHERENT |      \
                                VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)

+#define VM_WIMG_POSTED         VM_WIMG_IO

 #define VM_WIMG_WTHRU          (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 /* write combining mode, aka store gather */
 #define VM_WIMG_WCOMB          (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) 
 #define VM_WIMG_WTHRU          (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 /* write combining mode, aka store gather */
 #define VM_WIMG_WCOMB          (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) 


@@ -307,7 +308,17 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 /* This is conservative, but suffices */
 #define INTEL_PTE_RSVD         ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
 
 /* This is conservative, but suffices */
 #define INTEL_PTE_RSVD         ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
 

-#define INTEL_COMPRESSED       (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
+#define INTEL_PTE_COMPRESSED   (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
+#define INTEL_PTE_COMPRESSED_ALT (1ULL << 61) /* compressed but with "alternate accounting" */
+
+#define INTEL_PTE_COMPRESSED_MASK (INTEL_PTE_COMPRESSED | \
+                                  INTEL_PTE_COMPRESSED_ALT)
+#define PTE_IS_COMPRESSED(x)                                           \
+       ((((x) & INTEL_PTE_VALID) == 0) && /* PTE is not valid... */    \
+        ((x) & INTEL_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
+        ((!((x) & ~INTEL_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
+         (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
+                &(x), (x), (x) & ~INTEL_PTE_COMPRESSED_MASK), FALSE)))

 
 #define        pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 #define        pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 
 #define        pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 #define        pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */


@@ -407,7 +418,8 @@ extern boolean_t pmap_ept_support_ad;
 #define PTE_READ(is_ept)       ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
 #define PTE_WRITE(is_ept)      ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
 #define PTE_PS                 INTEL_PTE_PS
 #define PTE_READ(is_ept)       ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
 #define PTE_WRITE(is_ept)      ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
 #define PTE_PS                 INTEL_PTE_PS

-#define PTE_COMPRESSED         INTEL_COMPRESSED
+#define PTE_COMPRESSED         INTEL_PTE_COMPRESSED
+#define PTE_COMPRESSED_ALT     INTEL_PTE_COMPRESSED_ALT

 #define PTE_NCACHE(is_ept)     ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
 #define PTE_WTHRU(is_ept)      ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
 #define PTE_REF(is_ept)        ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)
 #define PTE_NCACHE(is_ept)     ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
 #define PTE_WTHRU(is_ept)      ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
 #define PTE_REF(is_ept)        ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)


@@ -434,7 +446,6 @@ extern pt_entry_t   *PTmap;
 extern pdpt_entry_t    *IdlePDPT;
 extern pml4_entry_t    *IdlePML4;
 extern boolean_t       no_shared_cr3;
 extern pdpt_entry_t    *IdlePDPT;
 extern pml4_entry_t    *IdlePML4;
 extern boolean_t       no_shared_cr3;

-extern addr64_t                kernel64_cr3;

 extern pd_entry_t      *IdlePTD;       /* physical addr of "Idle" state PTD */
 
 extern uint64_t                pmap_pv_hashlist_walks;
 extern pd_entry_t      *IdlePTD;       /* physical addr of "Idle" state PTD */
 
 extern uint64_t                pmap_pv_hashlist_walks;


@@ -442,9 +453,6 @@ extern uint64_t             pmap_pv_hashlist_cnts;
 extern uint32_t                pmap_pv_hashlist_max;
 extern uint32_t                pmap_kernel_text_ps;
 
 extern uint32_t                pmap_pv_hashlist_max;
 extern uint32_t                pmap_kernel_text_ps;
 

-
-
-#ifdef __x86_64__

 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 
 extern uint64_t physmap_base, physmap_max;
 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 
 extern uint64_t physmap_base, physmap_max;


@@ -466,6 +474,30 @@ static     inline void * PHYSMAP_PTOV_check(void *paddr) {
 }
 
 #define PHYSMAP_PTOV(x)        (PHYSMAP_PTOV_check((void*) (x)))
 }
 
 #define PHYSMAP_PTOV(x)        (PHYSMAP_PTOV_check((void*) (x)))

+#if MACH_KERNEL_PRIVATE
+extern uint64_t dblmap_base, dblmap_max, dblmap_dist;
+
+static inline uint64_t DBLMAP_CHECK(uintptr_t x) {
+       uint64_t dbladdr = (uint64_t)x + dblmap_dist;
+       if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+               panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+                   (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+       }
+       return dbladdr;
+
+}
+#define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x))
+extern uint64_t ldt_alias_offset;
+static inline uint64_t LDTALIAS_CHECK(uintptr_t x) {
+       uint64_t dbladdr = (uint64_t)x + ldt_alias_offset;
+       if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+               panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+                   (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+       }
+       return dbladdr;
+}
+#define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x))
+#endif

 
 /*
  * For KASLR, we alias the master processor's IDT and GDT at fixed
 
 /*
  * For KASLR, we alias the master processor's IDT and GDT at fixed


@@ -473,12 +505,13 @@ static    inline void * PHYSMAP_PTOV_check(void *paddr) {
  * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
  * The low global vector page is mapped at a fixed alias also.
  */
  * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
  * The low global vector page is mapped at a fixed alias also.
  */

-#define MASTER_IDT_ALIAS       (VM_MIN_KERNEL_ADDRESS + 0x0000)
-#define MASTER_GDT_ALIAS       (VM_MIN_KERNEL_ADDRESS + 0x1000)

 #define LOWGLOBAL_ALIAS                (VM_MIN_KERNEL_ADDRESS + 0x2000)
 #define LOWGLOBAL_ALIAS                (VM_MIN_KERNEL_ADDRESS + 0x2000)

-#define CPU_GDT_ALIAS(_cpu)    (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))

 
 

-#endif /*__x86_64__ */
+/*
+ * This indicates (roughly) where there is free space for the VM
+ * to use for the heap; this does not need to be precise.
+ */
+#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS

 
 #include <vm/vm_page.h>
 
 
 #include <vm/vm_page.h>
 


@@ -490,23 +523,28 @@ static    inline void * PHYSMAP_PTOV_check(void *paddr) {
 
 struct pmap {
        decl_simple_lock_data(,lock)    /* lock on map */
 
 struct pmap {
        decl_simple_lock_data(,lock)    /* lock on map */

-       pmap_paddr_t    pm_cr3;         /* physical addr */
-       pmap_paddr_t    pm_eptp;        /* EPTP */
-       boolean_t       pm_shared;
-        pd_entry_t      *dirbase;        /* page directory pointer */
-        vm_object_t     pm_obj;         /* object to hold pde's */
+       pmap_paddr_t    pm_cr3;         /* Kernel+user shared PML4 physical*/
+       pmap_paddr_t    pm_ucr3;        /* Mirrored user PML4 physical */

         task_map_t      pm_task_map;
         task_map_t      pm_task_map;

-        pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
-       pml4_entry_t    *pm_pml4;       /* VKA of top level */
-       vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
-       vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
+       boolean_t       pm_shared;
+       boolean_t       pagezero_accessible;

 #define        PMAP_PCID_MAX_CPUS      MAX_CPUS        /* Must be a multiple of 8 */
        pcid_t          pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
        volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
        struct pmap_statistics  stats;  /* map statistics */
        int             ref_count;      /* reference count */
         int            nx_enabled;
 #define        PMAP_PCID_MAX_CPUS      MAX_CPUS        /* Must be a multiple of 8 */
        pcid_t          pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
        volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
        struct pmap_statistics  stats;  /* map statistics */
        int             ref_count;      /* reference count */
         int            nx_enabled;

+       pml4_entry_t    *pm_pml4;       /* VKA of top level */
+       pml4_entry_t    *pm_upml4;      /* Shadow VKA of top level */
+       vm_object_t     pm_obj;         /* object to hold pde's */
+       vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
+       vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
+       pmap_paddr_t    pm_eptp;        /* EPTP */

        ledger_t        ledger;         /* ledger tracking phys mappings */
        ledger_t        ledger;         /* ledger tracking phys mappings */

+#if MACH_ASSERT
+       int             pmap_pid;
+       char            pmap_procname[17];
+#endif /* MACH_ASSERT */

 };
 
 static inline boolean_t
 };
 
 static inline boolean_t


@@ -571,22 +609,48 @@ extern pmap_memory_region_t pmap_memory_regions[];
 #include <i386/pmap_pcid.h>
 
 static inline void
 #include <i386/pmap_pcid.h>
 
 static inline void

-set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) {
+set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {

        int ccpu = my_cpu;
        int ccpu = my_cpu;

-       cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
+       uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3;
+       cpu_datap(ccpu)->cpu_task_cr3 = pcr3;
+       cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3;
+
+       cpu_datap(ccpu)->cpu_ucr3 = ucr3;
+       cpu_shadowp(ccpu)->cpu_ucr3 = ucr3;
+

        cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
        cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;

+
+       assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));
+       assert(ccpu == cpu_number());

        /*
         * Switch cr3 if necessary
         * - unless running with no_shared_cr3 debugging mode
         *   and we're not on the kernel's cr3 (after pre-empted copyio)
         */
        /*
         * Switch cr3 if necessary
         * - unless running with no_shared_cr3 debugging mode
         *   and we're not on the kernel's cr3 (after pre-empted copyio)
         */

+       boolean_t nopagezero = tpmap->pagezero_accessible;
+       boolean_t priorpagezero = cpu_datap(ccpu)->cpu_pagezero_mapped;
+       cpu_datap(ccpu)->cpu_pagezero_mapped = nopagezero;
+

        if (__probable(!no_shared_cr3)) {
        if (__probable(!no_shared_cr3)) {

-               if (get_cr3_base() != tpmap->pm_cr3) {
+               if (__improbable(nopagezero)) {
+                       boolean_t copyio_active = ((thread->machine.specFlags & CopyIOActive) != 0);

                        if (pmap_pcid_ncpus) {
                        if (pmap_pcid_ncpus) {

-                               pmap_pcid_activate(tpmap, ccpu);
+                               pmap_pcid_activate(tpmap, ccpu, TRUE, copyio_active);
+                       } else {
+                               if (copyio_active) {
+                                       if (get_cr3_base() != tpmap->pm_cr3) {
+                                               set_cr3_raw(tpmap->pm_cr3);
+                                       }
+                               } else if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) {
+                                       set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
+                               }

                        }
                        }

-                       else
+               } else if ((get_cr3_base() != tpmap->pm_cr3) || priorpagezero) {
+                       if (pmap_pcid_ncpus) {
+                               pmap_pcid_activate(tpmap, ccpu, FALSE, FALSE);
+                       } else {

                                set_cr3_raw(tpmap->pm_cr3);
                                set_cr3_raw(tpmap->pm_cr3);

+                       }

                }
        } else {
                if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
                }
        } else {
                if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)


@@ -601,10 +665,6 @@ set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) {
 extern void            process_pmap_updates(void);
 extern void            pmap_update_interrupt(void);
 
 extern void            process_pmap_updates(void);
 extern void            pmap_update_interrupt(void);
 

-/*
- *     Machine dependent routines that are used only for i386/i486/i860.
- */
-

 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 
 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 


@@ -612,23 +672,6 @@ extern kern_return_t       pmap_expand(
                                pmap_t          pmap,
                                vm_map_offset_t addr,
                                unsigned int options);
                                pmap_t          pmap,
                                vm_map_offset_t addr,
                                unsigned int options);

-#if    !defined(__x86_64__)
-extern pt_entry_t      *pmap_pte(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pd_entry_t      *pmap_pde(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pd_entry_t      *pmap64_pde(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pdpt_entry_t    *pmap64_pdpt(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-#endif

 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,
 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,


@@ -642,7 +685,6 @@ extern vm_offset_t  pmap_map_bd(
                                vm_map_offset_t end,
                                vm_prot_t       prot,
                                unsigned int    flags);
                                vm_map_offset_t end,
                                vm_prot_t       prot,
                                unsigned int    flags);

-

 extern void            pmap_bootstrap(
                                vm_offset_t     load_start,
                                boolean_t       IA32e);
 extern void            pmap_bootstrap(
                                vm_offset_t     load_start,
                                boolean_t       IA32e);


@@ -666,15 +708,15 @@ extern void               pmap_cpu_free(
                                struct cpu_pmap *cp);
 #endif
 
                                struct cpu_pmap *cp);
 #endif
 

-extern void            pmap_map_block(
-                               pmap_t pmap, 
+extern kern_return_t   pmap_map_block(
+                               pmap_t pmap,

                                addr64_t va,
                                ppnum_t pa,
                                uint32_t size,
                                vm_prot_t prot,
                                int attr,
                                unsigned int flags);
                                addr64_t va,
                                ppnum_t pa,
                                uint32_t size,
                                vm_prot_t prot,
                                int attr,
                                unsigned int flags);

-                               
+

 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);


@@ -683,7 +725,7 @@ extern void pmap_cpu_init(void);
 extern void pmap_disable_NX(pmap_t pmap);
 
 extern void pt_fake_zone_init(int);
 extern void pmap_disable_NX(pmap_t pmap);
 
 extern void pt_fake_zone_init(int);

-extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, 
+extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,

                              uint64_t *, int *, int *, int *);
 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
 
                              uint64_t *, int *, int *, int *);
 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
 


@@ -694,9 +736,9 @@ extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__pr
 
 #include <kern/spl.h>
 
 
 #include <kern/spl.h>
 

-                                 
+

 #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) {                               \
 #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) {                               \

-       register pmap_t         tpmap;                                  \
+       pmap_t          tpmap;                                  \

                                                                         \
         tpmap = vm_map_pmap(map);                                      \
         set_dirbase(tpmap, thread, my_cpu);                                    \
                                                                         \
         tpmap = vm_map_pmap(map);                                      \
         set_dirbase(tpmap, thread, my_cpu);                                    \


@@ -704,20 +746,11 @@ extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__pr
 
 #if   defined(__x86_64__)
 #define PMAP_DEACTIVATE_MAP(map, thread, ccpu)                         \
 
 #if   defined(__x86_64__)
 #define PMAP_DEACTIVATE_MAP(map, thread, ccpu)                         \

-       pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE);
+       pmap_assert2((pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE),"PCIDs: 0x%x, active PCID: 0x%x, CR3: 0x%lx, pmap_cr3: 0x%llx, kernel_cr3: 0x%llx, kernel pmap cr3: 0x%llx, CPU active PCID: 0x%x, CPU kernel PCID: 0x%x, specflags: 0x%x, pagezero: 0x%x", pmap_pcid_ncpus, pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu), get_cr3_raw(), map->pmap->pm_cr3, cpu_datap(ccpu)->cpu_kernel_cr3, kernel_pmap->pm_cr3, cpu_datap(ccpu)->cpu_active_pcid, cpu_datap(ccpu)->cpu_kernel_pcid, thread->machine.specFlags, map->pmap->pagezero_accessible);

 #else
 #define PMAP_DEACTIVATE_MAP(map, thread)
 #endif
 
 #else
 #define PMAP_DEACTIVATE_MAP(map, thread)
 #endif
 

-#define        PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
-                                                                        \
-       pmap_assert(ml_get_interrupts_enabled() == FALSE);              \
-       if (old_th->map != new_th->map) {                               \
-               PMAP_DEACTIVATE_MAP(old_th->map, old_th, my_cpu);               \
-               PMAP_ACTIVATE_MAP(new_th->map, new_th, my_cpu);         \
-       }                                                               \
-}
-

 #if NCOPY_WINDOWS > 0
 #define        PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
        spl_t           spl;                                            \
 #if NCOPY_WINDOWS > 0
 #define        PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
        spl_t           spl;                                            \


@@ -821,10 +854,15 @@ extern boolean_t pmap_is_empty(pmap_t             pmap,
 kern_return_t
 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
 
 kern_return_t
 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
 

+#if MACH_ASSERT
+extern int pmap_stats_assert;
+#define PMAP_STATS_ASSERTF(args)               \
+       MACRO_BEGIN                             \
+       if (pmap_stats_assert) assertf args;    \
+       MACRO_END
+#else /* MACH_ASSERT */
+#define PMAP_STATS_ASSERTF(args)
+#endif /* MACH_ASSERT */

 #endif /* ASSEMBLER */
 #endif /* ASSEMBLER */

-
-

 #endif /* _PMAP_MACHINE_ */
 #endif /* _PMAP_MACHINE_ */

-
-

 #endif  /* KERNEL_PRIVATE */
 #endif  /* KERNEL_PRIVATE */