xnu-1699.22.73.tar.gz

[apple/xnu.git] / osfmk / i386 / pmap.h

diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h
index 8a50070d4162f757dd87ded317fa68cab11b62ef..44b6bf7423272a0ccc3f21b6a4b82f0e747c00c3 100644 (file)
--- a/osfmk/i386/pmap.h
+++ b/osfmk/i386/pmap.h
@@ -1,6 +1,5 @@
-

 /*
 /*

- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -81,10 +80,13 @@
 #include <kern/kern_types.h>
 #include <kern/thread.h>
 #include <kern/lock.h>
 #include <kern/kern_types.h>
 #include <kern/thread.h>
 #include <kern/lock.h>

+#include <mach/branch_predicates.h>

 
 #include <i386/mp.h>
 #include <i386/proc_reg.h>
 
 
 #include <i386/mp.h>
 #include <i386/proc_reg.h>
 

+#include <i386/pal_routines.h>
+

 /*
  *     Define the generic in terms of the specific
  */
 /*
  *     Define the generic in terms of the specific
  */


@@ -112,10 +114,20 @@
 
 #define PTESHIFT        12
 
 
 #define PTESHIFT        12
 

+
+#define INITPT_SEG_BASE  0x100000
+#define INITGDT_SEG_BASE 0x106000
+#define SLEEP_SEG_BASE   0x107000
+
+#ifdef __x86_64__
+#define LOW_4GB_MASK   ((vm_offset_t)0x00000000FFFFFFFFUL)
+#endif
+

 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */
 
 #define INTEL_OFFMASK  (I386_PGBYTES - 1)
 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */
 
 #define INTEL_OFFMASK  (I386_PGBYTES - 1)

+#define INTEL_LOFFMASK (I386_LPGBYTES - 1)

 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))


@@ -126,6 +138,8 @@
 #define NBPDE           (1 << PDESHIFT)
 #define PDEMASK         (NBPDE - 1)
 
 #define NBPDE           (1 << PDESHIFT)
 #define PDEMASK         (NBPDE - 1)
 

+#define PTE_PER_PAGE   512 /* number of PTE's per page on any level */
+

  /* cleanly define parameters for all the page table levels */
 typedef uint64_t        pml4_entry_t;
 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
  /* cleanly define parameters for all the page table levels */
 typedef uint64_t        pml4_entry_t;
 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))


@@ -161,12 +175,38 @@ typedef uint64_t        pt_entry_t;
 
 typedef uint64_t  pmap_paddr_t;
 
 
 typedef uint64_t  pmap_paddr_t;
 

+#if    DEBUG
+#define PMAP_ASSERT 1
+#endif
+#if PMAP_ASSERT
+#define        pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+
+#define pmap_assert2(ex, fmt, args...)                                 \
+       do {                                                            \
+               if (!(ex)) {                                            \
+                       kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args); \
+                       panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args);           \
+               }                                                       \
+       } while(0)
+#else
+#define pmap_assert(ex)
+#define pmap_assert2(ex, fmt, args...)
+#endif
+
+/* superpages */
+#ifdef __x86_64__
+#define SUPERPAGE_NBASEPAGES 512
+#else
+#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
+#endif
+

 /*
  * Atomic 64-bit store of a page table entry.
  */
 static inline void
 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 {
 /*
  * Atomic 64-bit store of a page table entry.
  */
 static inline void
 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 {

+#ifdef __i386__

        /*
         * Load the new value into %ecx:%ebx
         * Load the old value into %edx:%eax
        /*
         * Load the new value into %ecx:%ebx
         * Load the old value into %edx:%eax


@@ -174,7 +214,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
         * If the compare succeeds, the new value will have been stored.
         * Otherwise, the old value changed and reloaded, so try again.
         */
         * If the compare succeeds, the new value will have been stored.
         * Otherwise, the old value changed and reloaded, so try again.
         */

-       asm volatile(
+       __asm__ volatile(

                "       movl    (%0), %%eax     \n\t"
                "       movl    4(%0), %%edx    \n\t"
                "1:                             \n\t"
                "       movl    (%0), %%eax     \n\t"
                "       movl    4(%0), %%edx    \n\t"
                "1:                             \n\t"


@@ -185,14 +225,74 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
                  "b" ((uint32_t)value),
                  "c" ((uint32_t)(value >> 32))
                : "eax", "edx", "memory");
                  "b" ((uint32_t)value),
                  "c" ((uint32_t)(value >> 32))
                : "eax", "edx", "memory");

+#else
+       /*
+        * In the 32-bit kernel a compare-and-exchange loop was
+        * required to provide atomicity. For K64, life is easier:
+        */
+       *entryp = value;
+#endif
+}
+
+/*
+ * Atomic 64-bit compare and exchange of a page table entry.
+ */
+static inline boolean_t
+pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
+{
+       boolean_t               ret;
+
+#ifdef __i386__
+       /*
+        * Load the old value into %edx:%eax
+        * Load the new value into %ecx:%ebx
+        * Compare-exchange-8bytes at address entryp (loaded in %edi)
+        * If the compare succeeds, the new value is stored, return TRUE.
+        * Otherwise, no swap is made, return FALSE.
+        */
+       asm volatile(
+               "       lock; cmpxchg8b (%1)    \n\t"
+               "       setz    %%al            \n\t"
+               "       movzbl  %%al,%0"
+               : "=a" (ret)
+               : "D" (entryp),
+                 "a" ((uint32_t)old),
+                 "d" ((uint32_t)(old >> 32)),
+                 "b" ((uint32_t)new),
+                 "c" ((uint32_t)(new >> 32))
+               : "memory");
+#else
+       /*
+        * Load the old value into %rax
+        * Load the new value into another register
+        * Compare-exchange-quad at address entryp
+        * If the compare succeeds, the new value is stored, return TRUE.
+        * Otherwise, no swap is made, return FALSE.
+        */
+       asm volatile(
+               "       lock; cmpxchgq %2,(%3)  \n\t"
+               "       setz    %%al            \n\t"
+               "       movzbl  %%al,%0"
+               : "=a" (ret)
+               : "a" (old),
+                 "r" (new),
+                 "r" (entryp)
+               : "memory");
+#endif
+       return ret;

 }
 
 }
 

+#define pmap_update_pte(entryp, old, new) \
+       while (!pmap_cmpx_pte((entryp), (old), (new)))
+
+

 /* in 64 bit spaces, the number of each type of page in the page tables */
 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 
 /* in 64 bit spaces, the number of each type of page in the page tables */
 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 

+#ifdef __i386__

 /*
  * The 64-bit kernel is remapped in uber-space which is at the base
  * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
 /*
  * The 64-bit kernel is remapped in uber-space which is at the base
  * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,


@@ -201,6 +301,13 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 #define KERNEL_UBER_PML4_INDEX 511
 #define KERNEL_UBER_BASE       (0ULL - NBPML4)
 #define KERNEL_UBER_BASE_HI32  ((uint32_t)(KERNEL_UBER_BASE >> 32))
 #define KERNEL_UBER_PML4_INDEX 511
 #define KERNEL_UBER_BASE       (0ULL - NBPML4)
 #define KERNEL_UBER_BASE_HI32  ((uint32_t)(KERNEL_UBER_BASE >> 32))

+#else
+#define KERNEL_PML4_INDEX      511
+#define KERNEL_KEXTS_INDEX     510     /* Home of KEXTs - the basement */
+#define KERNEL_PHYSMAP_INDEX   509     /* virtual to physical map */ 
+#define KERNEL_BASE            (0ULL - NBPML4)
+#define KERNEL_BASEMENT                (KERNEL_BASE - NBPML4)
+#endif

 
 #define        VM_WIMG_COPYBACK        VM_MEM_COHERENT
 #define        VM_WIMG_DEFAULT         VM_MEM_COHERENT
 
 #define        VM_WIMG_COPYBACK        VM_MEM_COHERENT
 #define        VM_WIMG_DEFAULT         VM_MEM_COHERENT


@@ -214,8 +321,18 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 /*
  * Pte related macros
  */
 /*
  * Pte related macros
  */

+#ifdef __i386__

 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))

+#else
+#define KVADDR(pmi, pdpi, pdi, pti)              \
+        ((vm_offset_t)                   \
+               ((uint64_t) -1    << 47)        | \
+               ((uint64_t)(pmi)  << PML4SHIFT) | \
+               ((uint64_t)(pdpi) << PDPTSHIFT) | \
+               ((uint64_t)(pdi)  << PDESHIFT)  | \
+               ((uint64_t)(pti)  << PTESHIFT))
+#endif

 
 /*
  * Size of Kernel address space.  This is the number of page table pages
 
 /*
  * Size of Kernel address space.  This is the number of page table pages


@@ -234,6 +351,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 #endif
 
 
 #endif
 
 

+#ifdef __i386__

 enum high_cpu_types {
   HIGH_CPU_ISS0,
   HIGH_CPU_ISS1,
 enum high_cpu_types {
   HIGH_CPU_ISS0,
   HIGH_CPU_ISS1,


@@ -279,16 +397,20 @@ enum  high_fixed_addresses {
 
 #define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
 #define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
 
 #define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
 #define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))

+#endif

 
 /*
  *     Convert address offset to page descriptor index
  */
 
 /*
  *     Convert address offset to page descriptor index
  */

-#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
+#define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
+#define pdenum(pmap, a)        (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
+#define PMAP_INVALID_PDPTNUM (~0ULL)

 
 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 
 
 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 

+

 /*
  *     Convert page descriptor index to user virtual address
  */
 /*
  *     Convert page descriptor index to user virtual address
  */


@@ -312,15 +434,18 @@ enum  high_fixed_addresses {
 #define INTEL_PTE_NCACHE       0x00000010
 #define INTEL_PTE_REF          0x00000020
 #define INTEL_PTE_MOD          0x00000040
 #define INTEL_PTE_NCACHE       0x00000010
 #define INTEL_PTE_REF          0x00000020
 #define INTEL_PTE_MOD          0x00000040

-#define INTEL_PTE_PS            0x00000080
-#define INTEL_PTE_GLOBAL        0x00000100
+#define INTEL_PTE_PS           0x00000080
+#define INTEL_PTE_PTA          0x00000080
+#define INTEL_PTE_GLOBAL       0x00000100

 #define INTEL_PTE_WIRED                0x00000200
 #define INTEL_PTE_WIRED                0x00000200

+#define INTEL_PDPTE_NESTED     0x00000400

 #define INTEL_PTE_PFN          PG_FRAME
 #define INTEL_PTE_PFN          PG_FRAME

-#define INTEL_PTE_PTA          0x00000080

 
 #define INTEL_PTE_NX           (1ULL << 63)
 
 #define INTEL_PTE_INVALID       0
 
 #define INTEL_PTE_NX           (1ULL << 63)
 
 #define INTEL_PTE_INVALID       0

+/* This is conservative, but suffices */
+#define INTEL_PTE_RSVD         ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))

 
 #define        pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 #define        pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 
 #define        pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 #define        pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */


@@ -347,27 +472,43 @@ enum  high_fixed_addresses {
  * and directories.
  */
 
  * and directories.
  */
 

-extern pt_entry_t PTmap[], APTmap[], Upte;
-extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
-
-extern pd_entry_t *IdlePTD;    /* physical address of "Idle" state directory */
-extern pdpt_entry_t *IdlePDPT;
-
-extern pmap_paddr_t lo_kernel_cr3;
-
-extern pml4_entry_t *IdlePML4;
-extern pdpt_entry_t *IdlePDPT64;
-extern addr64_t     kernel64_cr3;
-extern boolean_t    no_shared_cr3;
-
+#ifdef __i386__
+extern pt_entry_t      PTmap[], APTmap[], Upte;
+extern pd_entry_t      PTD[], APTD[], PTDpde[], APTDpde[], Upde;
+extern pmap_paddr_t    lo_kernel_cr3;
+extern pdpt_entry_t    *IdlePDPT64;
+#else
+extern pt_entry_t      *PTmap;
+#endif
+extern boolean_t       no_shared_cr3;
+extern addr64_t                kernel64_cr3;
+extern pd_entry_t      *IdlePTD;       /* physical addr of "Idle" state PTD */
+extern pdpt_entry_t    IdlePDPT[];
+extern pml4_entry_t    IdlePML4[];
+
+extern uint64_t                pmap_pv_hashlist_walks;
+extern uint64_t                pmap_pv_hashlist_cnts;
+extern uint32_t                pmap_pv_hashlist_max;
+extern uint32_t                pmap_kernel_text_ps;
+
+#ifdef __i386__

 /*
 /*

+ * ** i386 **

  * virtual address to page table entry and
  * to physical address. Likewise for alternate address space.
  * Note: these work recursively, thus vtopte of a pte will give
  * the corresponding pde that in turn maps it.
  */
  * virtual address to page table entry and
  * to physical address. Likewise for alternate address space.
  * Note: these work recursively, thus vtopte of a pte will give
  * the corresponding pde that in turn maps it.
  */

+

 #define        vtopte(va)      (PTmap + i386_btop((vm_offset_t)va))
 #define        vtopte(va)      (PTmap + i386_btop((vm_offset_t)va))

+#endif
+
+#ifdef __x86_64__
+#define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))

 
 

+#define PHYSMAP_BASE   KVADDR(KERNEL_PHYSMAP_INDEX,0,0,0)
+#define PHYSMAP_PTOV(x)        ((void *)(((uint64_t)(x)) + PHYSMAP_BASE))
+#endif

 
 typedef        volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
                                        /* changed by other processors */
 
 typedef        volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
                                        /* changed by other processors */


@@ -385,26 +526,30 @@ struct md_page {
  */
 
 struct pmap {
  */
 
 struct pmap {

+       decl_simple_lock_data(,lock)    /* lock on map */
+       pmap_paddr_t    pm_cr3;         /* physical addr */
+       boolean_t       pm_shared;

         pd_entry_t      *dirbase;        /* page directory pointer */
         pd_entry_t      *dirbase;        /* page directory pointer */

+#ifdef __i386__

        pmap_paddr_t    pdirbase;        /* phys. address of dirbase */
        pmap_paddr_t    pdirbase;        /* phys. address of dirbase */

-        vm_object_t     pm_obj;         /* object to hold pde's */
-       int             ref_count;      /* reference count */
-        int            nx_enabled;
-        boolean_t       pm_64bit;
-        boolean_t       pm_kernel_cr3;
-        boolean_t       pm_shared;
-       decl_simple_lock_data(,lock)    /* lock on map */
-       struct pmap_statistics  stats;  /* map statistics */

        vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
        vm_offset_t     pm_hold;        /* true pdpt zalloc addr */

-       pmap_paddr_t    pm_cr3;         /* physical addr */
+#endif
+        vm_object_t     pm_obj;         /* object to hold pde's */
+        task_map_t      pm_task_map;

         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
        pml4_entry_t    *pm_pml4;       /* VKA of top level */
        vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
        vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
        pml4_entry_t    *pm_pml4;       /* VKA of top level */
        vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
        vm_object_t     pm_obj_pml4;    /* holds pml4 pages */

-       vm_object_t     pm_obj_top;     /* holds single top level page */
+#define        PMAP_PCID_MAX_CPUS      (48)    /* Must be a multiple of 8 */
+       pcid_t          pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
+       volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
+       struct pmap_statistics  stats;  /* map statistics */
+       int             ref_count;      /* reference count */
+        int            nx_enabled;

 };
 
 
 };
 
 

+#if NCOPY_WINDOWS > 0

 #define PMAP_PDPT_FIRST_WINDOW 0
 #define PMAP_PDPT_NWINDOWS 4
 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
 #define PMAP_PDPT_FIRST_WINDOW 0
 #define PMAP_PDPT_NWINDOWS 4
 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)


@@ -430,6 +575,8 @@ typedef struct cpu_pmap {
 
 
 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 
 
 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);

+extern void         pmap_put_mapwindow(mapwindow_t *map);
+#endif

 
 typedef struct pmap_memory_regions {
   ppnum_t base;
 
 typedef struct pmap_memory_regions {
   ppnum_t base;


@@ -438,16 +585,38 @@ typedef struct pmap_memory_regions {
   uint32_t type;
 } pmap_memory_region_t;
 
   uint32_t type;
 } pmap_memory_region_t;
 

-unsigned pmap_memory_region_count;
-unsigned pmap_memory_region_current;
+extern unsigned pmap_memory_region_count;
+extern unsigned pmap_memory_region_current;

 
 #define PMAP_MEMORY_REGIONS_SIZE 128
 
 extern pmap_memory_region_t pmap_memory_regions[];
 
 #define PMAP_MEMORY_REGIONS_SIZE 128
 
 extern pmap_memory_region_t pmap_memory_regions[];

+#include <i386/pmap_pcid.h>

 
 

-static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
-       current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
-       current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
+static inline void
+set_dirbase(pmap_t tpmap, __unused thread_t thread) {
+       int ccpu = cpu_number();
+       cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
+       cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
+#ifndef __i386__
+       /*
+        * Switch cr3 if necessary
+        * - unless running with no_shared_cr3 debugging mode
+        *   and we're not on the kernel's cr3 (after pre-empted copyio)
+        */
+       if (__probable(!no_shared_cr3)) {
+               if (get_cr3_base() != tpmap->pm_cr3) {
+                       if (pmap_pcid_ncpus) {
+                               pmap_pcid_activate(tpmap, ccpu);
+                       }
+                       else
+                               set_cr3_raw(tpmap->pm_cr3);
+               }
+       } else {
+               if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
+                       set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
+       }
+#endif

 }
 
 /*
 }
 
 /*


@@ -464,6 +633,10 @@ extern void                pmap_update_interrupt(void);
 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 
 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 

+extern void            pmap_expand(
+                               pmap_t          pmap,
+                               vm_map_offset_t addr);
+#if    !defined(__x86_64__)

 extern pt_entry_t      *pmap_pte(
                                struct pmap     *pmap,
                                vm_map_offset_t addr);
 extern pt_entry_t      *pmap_pte(
                                struct pmap     *pmap,
                                vm_map_offset_t addr);


@@ -479,7 +652,7 @@ extern pd_entry_t   *pmap64_pde(
 extern pdpt_entry_t    *pmap64_pdpt(
                                struct pmap     *pmap,
                                vm_map_offset_t addr);
 extern pdpt_entry_t    *pmap64_pdpt(
                                struct pmap     *pmap,
                                vm_map_offset_t addr);

-
+#endif

 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,
 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,


@@ -505,7 +678,8 @@ extern int          pmap_list_resident_pages(
                                struct pmap     *pmap,
                                vm_offset_t     *listp,
                                int             space);
                                struct pmap     *pmap,
                                vm_offset_t     *listp,
                                int             space);

-
+extern void            x86_filter_TLB_coherency_interrupts(boolean_t);
+#ifdef __i386__

 extern void             pmap_commpage32_init(
                                           vm_offset_t kernel,
                                           vm_offset_t user,
 extern void             pmap_commpage32_init(
                                           vm_offset_t kernel,
                                           vm_offset_t user,


@@ -515,10 +689,17 @@ extern void             pmap_commpage64_init(
                                           vm_map_offset_t user,
                                           int count);
 
                                           vm_map_offset_t user,
                                           int count);
 

+#endif
+/*
+ * Get cache attributes (as pagetable bits) for the specified phys page
+ */
+extern unsigned        pmap_get_cache_attributes(ppnum_t);
+#if NCOPY_WINDOWS > 0

 extern struct cpu_pmap *pmap_cpu_alloc(
                                boolean_t       is_boot_cpu);
 extern void            pmap_cpu_free(
                                struct cpu_pmap *cp);
 extern struct cpu_pmap *pmap_cpu_alloc(
                                boolean_t       is_boot_cpu);
 extern void            pmap_cpu_free(
                                struct cpu_pmap *cp);

+#endif

 
 extern void            pmap_map_block(
                                pmap_t pmap, 
 
 extern void            pmap_map_block(
                                pmap_t pmap, 


@@ -532,14 +713,10 @@ extern void               pmap_map_block(
 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);

-extern void pmap_sync_page_data_phys(ppnum_t pa);
-extern void pmap_sync_page_attributes_phys(ppnum_t pa);

 
 

-extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern void pmap_cpu_init(void);

 extern void pmap_disable_NX(pmap_t pmap);
 extern void pmap_disable_NX(pmap_t pmap);

+#ifdef __i386__

 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
 extern void pmap_load_kernel_cr3(void);
 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
 extern void pmap_load_kernel_cr3(void);


@@ -548,10 +725,12 @@ extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);

+#endif

 
 

-extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
-
-
+extern void pt_fake_zone_init(int);
+extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, 
+                             uint64_t *, int *, int *, int *);
+extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));

 
 /*
  *     Macros for speed.
 
 /*
  *     Macros for speed.


@@ -560,53 +739,26 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
 
 #include <kern/spl.h>
 
 
 #include <kern/spl.h>
 

-#if defined(PMAP_ACTIVATE_KERNEL)
-#undef PMAP_ACTIVATE_KERNEL
-#undef PMAP_DEACTIVATE_KERNEL
-#undef PMAP_ACTIVATE_USER
-#undef PMAP_DEACTIVATE_USER
-#endif
-
-
-#define PMAP_ACTIVATE_KERNEL(my_cpu)  {                                        \
-        spl_t          spl;                                            \
-                                                                        \
-        spl = splhigh();                                               \
-       if (current_cpu_datap()->cpu_tlb_invalid)                       \
-               process_pmap_updates();                                 \
-        splx(spl);                                                     \
-}
-
-#define PMAP_DEACTIVATE_KERNEL(my_cpu) {                               \
-        spl_t          spl;                                            \
-                                                                        \
-        spl = splhigh();                                               \
-       process_pmap_updates();                                         \
-        splx(spl);                                                     \
-}
-
-
-#define PMAP_ACTIVATE_MAP(map, my_cpu) {                               \
+                                 
+#define PMAP_ACTIVATE_MAP(map, thread) {                               \

        register pmap_t         tpmap;                                  \
                                                                         \
         tpmap = vm_map_pmap(map);                                      \
        register pmap_t         tpmap;                                  \
                                                                         \
         tpmap = vm_map_pmap(map);                                      \

-        set_dirbase(tpmap, my_cpu);                                    \
-}
-
-#define PMAP_DEACTIVATE_MAP(map, my_cpu)                                \
-        if (current_cpu_datap()->cpu_task_map == TASK_MAP_64BIT_SHARED) \
-            pmap_load_kernel_cr3();
-
-#define PMAP_ACTIVATE_USER(th, my_cpu)  {                              \
-        spl_t          spl;                                            \
-                                                                        \
-        spl = splhigh();                                               \
-        PMAP_ACTIVATE_MAP(th->map, my_cpu)                             \
-        splx(spl);                                                     \
+        set_dirbase(tpmap, thread);                                    \

 }
 
 }
 

-#define PMAP_DEACTIVATE_USER(th, my_cpu)
+#ifdef __i386__
+#define PMAP_DEACTIVATE_MAP(map, thread)                               \
+       if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED)     \
+               pmap_load_kernel_cr3();
+#elif defined(__x86_64__)
+#define PMAP_DEACTIVATE_MAP(map, thread)                               \
+       pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, cpu_number()) == (get_cr3_raw() & 0xFFF)) : TRUE);
+#else
+#define PMAP_DEACTIVATE_MAP(map, thread)
+#endif

 
 

+#if   defined(__i386__)

 
 #define        PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
        spl_t           spl;                                            \
 
 #define        PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
        spl_t           spl;                                            \


@@ -617,16 +769,16 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
                                                                         \
         need_flush = 0;                                                        \
         spl = splhigh();                                               \
                                                                         \
         need_flush = 0;                                                        \
         spl = splhigh();                                               \

-       if (old_th->map != new_th->map) {                               \
-               PMAP_DEACTIVATE_MAP(old_th->map, my_cpu);               \
-               PMAP_ACTIVATE_MAP(new_th->map, my_cpu);                 \
+       if ((old_th->map != new_th->map) || (new_th->task != old_th->task)) {   \
+               PMAP_DEACTIVATE_MAP(old_th->map, old_th);               \
+               PMAP_ACTIVATE_MAP(new_th->map, new_th);                 \

        }                                                               \
         kpdp = current_cpu_datap()->cpu_copywindow_pdp;                        \
         for (i = 0; i < NCOPY_WINDOWS; i++) {                          \
                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {     \
                        updp = pmap_pde(new_th->map->pmap,              \
                               new_th->machine.copy_window[i].user_base);\
        }                                                               \
         kpdp = current_cpu_datap()->cpu_copywindow_pdp;                        \
         for (i = 0; i < NCOPY_WINDOWS; i++) {                          \
                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {     \
                        updp = pmap_pde(new_th->map->pmap,              \
                               new_th->machine.copy_window[i].user_base);\

-                       pmap_store_pte(kpdp,  updp ? *updp : 0);        \
+                        pmap_store_pte(kpdp, updp ? *updp : 0);                \

                 }                                                      \
                 kpdp++;                                                        \
         }                                                              \
                 }                                                      \
                 kpdp++;                                                        \
         }                                                              \


@@ -636,8 +788,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
         else                                                           \
                 new_th->machine.copyio_state = WINDOWS_DIRTY;          \
         if (new_th->machine.physwindow_pte) {                          \
         else                                                           \
                 new_th->machine.copyio_state = WINDOWS_DIRTY;          \
         if (new_th->machine.physwindow_pte) {                          \

-                pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),     \
-                                new_th->machine.physwindow_pte);       \
+         pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),    \
+                              new_th->machine.physwindow_pte);         \

                 if (need_flush == 0)                                   \
                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
         }                                                              \
                 if (need_flush == 0)                                   \
                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
         }                                                              \


@@ -645,22 +797,45 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
                 flush_tlb();                                           \
 }
 
                 flush_tlb();                                           \
 }
 

+#else /* __x86_64__ */
+#define        PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
+                                                                        \
+       pmap_assert(ml_get_interrupts_enabled() == FALSE);              \
+       if (old_th->map != new_th->map) {                               \
+               PMAP_DEACTIVATE_MAP(old_th->map, old_th);               \
+               PMAP_ACTIVATE_MAP(new_th->map, new_th);                 \
+       }                                                               \
+}
+#endif /* __i386__ */
+
+#if NCOPY_WINDOWS > 0

 #define        PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
        spl_t           spl;                                            \
                                                                        \
        spl = splhigh();                                                \
 #define        PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
        spl_t           spl;                                            \
                                                                        \
        spl = splhigh();                                                \

-       PMAP_DEACTIVATE_MAP(th->map, my_cpu);                           \
+       PMAP_DEACTIVATE_MAP(th->map, th);                               \

        th->map = new_map;                                              \
        th->map = new_map;                                              \

-       PMAP_ACTIVATE_MAP(th->map, my_cpu);                             \
+       PMAP_ACTIVATE_MAP(th->map, th);                                 \

        splx(spl);                                                      \
        splx(spl);                                                      \

-        inval_copy_windows(th);                                                \
+       inval_copy_windows(th);                                         \

 }
 }

+#else
+#define        PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
+       spl_t           spl;                                            \
+                                                                       \
+       spl = splhigh();                                                \
+       PMAP_DEACTIVATE_MAP(th->map, th);                               \
+       th->map = new_map;                                              \
+       PMAP_ACTIVATE_MAP(th->map, th);                                 \
+       splx(spl);                                                      \
+}
+#endif

 
 /*
  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
  * Marking the current cpu's cr3 active once more involves clearng this bit.
  * Note that valid page tables are page-aligned and so the bottom 12 bits
 
 /*
  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
  * Marking the current cpu's cr3 active once more involves clearng this bit.
  * Note that valid page tables are page-aligned and so the bottom 12 bits

- * are noramlly zero.
+ * are normally zero, modulo PCID.

  * We can only mark the current cpu active/inactive but we can test any cpu.
  */
 #define CPU_CR3_MARK_INACTIVE()                                                \
  * We can only mark the current cpu active/inactive but we can test any cpu.
  */
 #define CPU_CR3_MARK_INACTIVE()                                                \


@@ -672,10 +847,28 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
 #define CPU_CR3_IS_ACTIVE(cpu)                                         \
        ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 
 #define CPU_CR3_IS_ACTIVE(cpu)                                         \
        ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 

-#define CPU_GET_ACTIVE_CR3(cpu)                                         \
-        (cpu_datap(cpu)->cpu_active_cr3 & ~1)
+#define CPU_GET_ACTIVE_CR3(cpu)                                                \
+       (cpu_datap(cpu)->cpu_active_cr3 & ~1)

 
 

+#define CPU_GET_TASK_CR3(cpu)                                          \
+       (cpu_datap(cpu)->cpu_task_cr3)
+
+/*
+ *     Mark this cpu idle, and remove it from the active set,
+ *     since it is not actively using any pmap.  Signal_cpus
+ *     will notice that it is idle, and avoid signaling it,
+ *     but will queue the update request for when the cpu
+ *     becomes active.
+ */
+#if   defined(__x86_64__)

 #define MARK_CPU_IDLE(my_cpu)  {                                       \
 #define MARK_CPU_IDLE(my_cpu)  {                                       \

+       assert(ml_get_interrupts_enabled() == FALSE);                   \
+       CPU_CR3_MARK_INACTIVE();                                        \
+       __asm__ volatile("mfence");                                     \
+}
+#else /* __i386__ native */
+#define MARK_CPU_IDLE(my_cpu)  {                                       \
+       assert(ml_get_interrupts_enabled() == FALSE);                   \

        /*                                                              \
         *      Mark this cpu idle, and remove it from the active set,  \
         *      since it is not actively using any pmap.  Signal_cpus   \
        /*                                                              \
         *      Mark this cpu idle, and remove it from the active set,  \
         *      since it is not actively using any pmap.  Signal_cpus   \


@@ -683,19 +876,17 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
         *      but will queue the update request for when the cpu      \
         *      becomes active.                                         \
         */                                                             \
         *      but will queue the update request for when the cpu      \
         *      becomes active.                                         \
         */                                                             \

-       int     s = splhigh();                                          \

        if (!cpu_mode_is64bit() || no_shared_cr3)                       \
                process_pmap_updates();                                 \
        else                                                            \
                pmap_load_kernel_cr3();                                 \
        CPU_CR3_MARK_INACTIVE();                                        \
        __asm__ volatile("mfence");                                     \
        if (!cpu_mode_is64bit() || no_shared_cr3)                       \
                process_pmap_updates();                                 \
        else                                                            \
                pmap_load_kernel_cr3();                                 \
        CPU_CR3_MARK_INACTIVE();                                        \
        __asm__ volatile("mfence");                                     \

-       splx(s);                                                        \

 }
 }

+#endif /* __i386__ */

 
 #define MARK_CPU_ACTIVE(my_cpu) {                                      \
 
 #define MARK_CPU_ACTIVE(my_cpu) {                                      \

-                                                                       \
-       int     s = splhigh();                                          \
+       assert(ml_get_interrupts_enabled() == FALSE);                   \

        /*                                                              \
         *      If a kernel_pmap update was requested while this cpu    \
         *      was idle, process it as if we got the interrupt.        \
        /*                                                              \
         *      If a kernel_pmap update was requested while this cpu    \
         *      was idle, process it as if we got the interrupt.        \


@@ -711,7 +902,6 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
                                                                        \
        if (current_cpu_datap()->cpu_tlb_invalid)                       \
            process_pmap_updates();                                     \
                                                                        \
        if (current_cpu_datap()->cpu_tlb_invalid)                       \
            process_pmap_updates();                                     \

-       splx(s);                                                        \

 }
 
 #define PMAP_CONTEXT(pmap, thread)
 }
 
 #define PMAP_CONTEXT(pmap, thread)


@@ -722,12 +912,21 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
 
 
 #define pmap_resident_count(pmap)      ((pmap)->stats.resident_count)
 
 
 #define pmap_resident_count(pmap)      ((pmap)->stats.resident_count)

+#define pmap_resident_max(pmap)                ((pmap)->stats.resident_max)

 #define        pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 #define        pmap_attribute(pmap,addr,size,attr,value) \
                                        (KERN_INVALID_ADDRESS)
 #define        pmap_attribute_cache_sync(addr,size,attr,value) \
                                        (KERN_INVALID_ADDRESS)
 
 #define        pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 #define        pmap_attribute(pmap,addr,size,attr,value) \
                                        (KERN_INVALID_ADDRESS)
 #define        pmap_attribute_cache_sync(addr,size,attr,value) \
                                        (KERN_INVALID_ADDRESS)
 

+#define MACHINE_PMAP_IS_EMPTY  1
+extern boolean_t pmap_is_empty(pmap_t          pmap,
+                              vm_map_offset_t  start,
+                              vm_map_offset_t  end);
+
+#define MACHINE_BOOTSTRAPPTD   1       /* Static bootstrap page-tables */
+
+

 #endif /* ASSEMBLER */
 
 
 #endif /* ASSEMBLER */