xnu-3789.1.32.tar.gz

[apple/xnu.git] / osfmk / i386 / pmap_internal.h

diff --git a/osfmk/i386/pmap_internal.h b/osfmk/i386/pmap_internal.h
index 6227e50f78239207384ae1f75bf9bdbc0858d735..3c8909968de9046e5682b111cb71d60a0f885fb8 100644 (file)
--- a/osfmk/i386/pmap_internal.h
+++ b/osfmk/i386/pmap_internal.h
@@ -219,7 +219,7 @@ than the original pv lists that contained all aliases for the specific ppn.
 typedef struct pv_rooted_entry {
        /* first three entries must match pv_hashed_entry_t */
         queue_head_t           qlink;
 typedef struct pv_rooted_entry {
        /* first three entries must match pv_hashed_entry_t */
         queue_head_t           qlink;

-       vm_map_offset_t         va;     /* virtual address for mapping */
+       vm_map_offset_t         va_and_flags;   /* virtual address for mapping */

        pmap_t                  pmap;   /* pmap where mapping lies */
 } *pv_rooted_entry_t;
 
        pmap_t                  pmap;   /* pmap where mapping lies */
 } *pv_rooted_entry_t;
 


@@ -228,7 +228,7 @@ typedef struct pv_rooted_entry {
 typedef struct pv_hashed_entry {
        /* first three entries must match pv_rooted_entry_t */
        queue_head_t            qlink;
 typedef struct pv_hashed_entry {
        /* first three entries must match pv_rooted_entry_t */
        queue_head_t            qlink;

-       vm_map_offset_t         va;
+       vm_map_offset_t         va_and_flags;

        pmap_t                  pmap;
        ppnum_t                 ppn;
        struct pv_hashed_entry  *nexth;
        pmap_t                  pmap;
        ppnum_t                 ppn;
        struct pv_hashed_entry  *nexth;


@@ -236,6 +236,12 @@ typedef struct pv_hashed_entry {
 
 #define PV_HASHED_ENTRY_NULL ((pv_hashed_entry_t)0)
 
 
 #define PV_HASHED_ENTRY_NULL ((pv_hashed_entry_t)0)
 

+#define PVE_VA(pve) ((pve)->va_and_flags & ~PAGE_MASK)
+#define PVE_FLAGS(pve) ((pve)->va_and_flags & PAGE_MASK)
+#define PVE_IS_ALTACCT 0x001
+#define PVE_IS_ALTACCT_PAGE(pve) \
+       (((pve)->va_and_flags & PVE_IS_ALTACCT) ? TRUE : FALSE)
+

 //#define PV_DEBUG 1   /* uncomment to enable some PV debugging code */
 #ifdef PV_DEBUG
 #define CHK_NPVHASH() if(0 == npvhashmask) panic("npvhash uninitialized");
 //#define PV_DEBUG 1   /* uncomment to enable some PV debugging code */
 #ifdef PV_DEBUG
 #define CHK_NPVHASH() if(0 == npvhashmask) panic("npvhash uninitialized");


@@ -379,22 +385,25 @@ static inline void pmap_pv_throttle(__unused pmap_t p) {
        (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_INTERNAL))
 #define IS_REUSABLE_PAGE(x)                    \
        (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_REUSABLE))
        (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_INTERNAL))
 #define IS_REUSABLE_PAGE(x)                    \
        (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_REUSABLE))

+#define IS_ALTACCT_PAGE(x)                             \
+       (IS_MANAGED_PAGE((x)) &&                        \
+        (PVE_IS_ALTACCT_PAGE(&pv_head_table[(x)])))

 
 /*
  *     Physical page attributes.  Copy bits from PTE definition.
  */
 #define        PHYS_MODIFIED   INTEL_PTE_MOD   /* page modified */
 #define        PHYS_REFERENCED INTEL_PTE_REF   /* page referenced */
 
 /*
  *     Physical page attributes.  Copy bits from PTE definition.
  */
 #define        PHYS_MODIFIED   INTEL_PTE_MOD   /* page modified */
 #define        PHYS_REFERENCED INTEL_PTE_REF   /* page referenced */

-#define PHYS_MANAGED   INTEL_PTE_VALID /* page is managed */
-#define PHYS_NOENCRYPT INTEL_PTE_USER  /* no need to encrypt this page in the hibernation image */
+#define        PHYS_MANAGED    INTEL_PTE_VALID /* page is managed */
+#define        PHYS_NOENCRYPT  INTEL_PTE_USER  /* no need to encrypt this page in the hibernation image */

 #define        PHYS_NCACHE     INTEL_PTE_NCACHE
 #define        PHYS_PTA        INTEL_PTE_PTA
 #define        PHYS_CACHEABILITY_MASK (INTEL_PTE_PTA | INTEL_PTE_NCACHE)
 #define        PHYS_NCACHE     INTEL_PTE_NCACHE
 #define        PHYS_PTA        INTEL_PTE_PTA
 #define        PHYS_CACHEABILITY_MASK (INTEL_PTE_PTA | INTEL_PTE_NCACHE)

-#define PHYS_INTERNAL  INTEL_PTE_WTHRU /* page from internal object */
-#define PHYS_REUSABLE  INTEL_PTE_WRITE /* page is "reusable" */
+#define        PHYS_INTERNAL   INTEL_PTE_WTHRU /* page from internal object */
+#define        PHYS_REUSABLE   INTEL_PTE_WRITE /* page is "reusable" */

 
 

-extern const boolean_t pmap_disable_kheap_nx;
-extern const boolean_t pmap_disable_kstack_nx;
+extern boolean_t       pmap_disable_kheap_nx;
+extern boolean_t       pmap_disable_kstack_nx;

 
 #define PMAP_EXPAND_OPTIONS_NONE (0x0)
 #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT)
 
 #define PMAP_EXPAND_OPTIONS_NONE (0x0)
 #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT)


@@ -501,7 +510,7 @@ pmap_pvh_unlink(pv_hashed_entry_t pvh)
        int                     pvhash_idx;
 
        CHK_NPVHASH();
        int                     pvhash_idx;
 
        CHK_NPVHASH();

-       pvhash_idx = pvhashidx(pvh->pmap, pvh->va);
+       pvhash_idx = pvhashidx(pvh->pmap, PVE_VA(pvh));

 
        pprevh = pvhash(pvhash_idx);
 
 
        pprevh = pvhash(pvhash_idx);
 


@@ -530,7 +539,7 @@ pv_hash_add(pv_hashed_entry_t       pvh_e,
        int                     pvhash_idx;
 
        CHK_NPVHASH();
        int                     pvhash_idx;
 
        CHK_NPVHASH();

-       pvhash_idx = pvhashidx(pvh_e->pmap, pvh_e->va);
+       pvhash_idx = pvhashidx(pvh_e->pmap, PVE_VA(pvh_e));

        LOCK_PV_HASH(pvhash_idx);
        insque(&pvh_e->qlink, &pv_h->qlink);
        hashp = pvhash(pvhash_idx);
        LOCK_PV_HASH(pvhash_idx);
        insque(&pvh_e->qlink, &pv_h->qlink);
        hashp = pvhash(pvhash_idx);


@@ -549,7 +558,7 @@ pv_hash_remove(pv_hashed_entry_t pvh_e)
        int                     pvhash_idx;
 
        CHK_NPVHASH();
        int                     pvhash_idx;
 
        CHK_NPVHASH();

-       pvhash_idx = pvhashidx(pvh_e->pmap,pvh_e->va);
+       pvhash_idx = pvhashidx(pvh_e->pmap,PVE_VA(pvh_e));

        LOCK_PV_HASH(pvhash_idx);
        remque(&pvh_e->qlink);
        pmap_pvh_unlink(pvh_e);
        LOCK_PV_HASH(pvhash_idx);
        remque(&pvh_e->qlink);
        pmap_pvh_unlink(pvh_e);


@@ -651,8 +660,10 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t *
        pv_rooted_entry_t       pv_e = pv_h;
        uint32_t        bitdex;
        pmap_t pvpmap = pv_h->pmap;
        pv_rooted_entry_t       pv_e = pv_h;
        uint32_t        bitdex;
        pmap_t pvpmap = pv_h->pmap;

-       vm_map_offset_t pvva = pv_h->va;
+       vm_map_offset_t pvva = PVE_VA(pv_h);
+       vm_map_offset_t pve_flags = PVE_FLAGS(pv_h);

        boolean_t ppcd = FALSE;
        boolean_t ppcd = FALSE;

+       boolean_t is_ept;

 
        /* Ideally, we'd consult the Mach VM here to definitively determine
         * the nature of the mapping for this address space and address.
 
        /* Ideally, we'd consult the Mach VM here to definitively determine
         * the nature of the mapping for this address space and address.


@@ -664,16 +675,21 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t *
 
        /* As a precautionary measure, mark A+D */
        pmap_phys_attributes[ppn_to_pai(ppn)] |= (PHYS_MODIFIED | PHYS_REFERENCED);
 
        /* As a precautionary measure, mark A+D */
        pmap_phys_attributes[ppn_to_pai(ppn)] |= (PHYS_MODIFIED | PHYS_REFERENCED);

+       is_ept = is_ept_pmap(pmap);

 
        /*
         * Correct potential single bit errors in either (but not both) element
         * of the PV
         */
        do {
 
        /*
         * Correct potential single bit errors in either (but not both) element
         * of the PV
         */
        do {

-               if ((popcnt1((uintptr_t)pv_e->pmap ^ (uintptr_t)pmap) && pv_e->va == vaddr) ||
-                   (pv_e->pmap == pmap && popcnt1(pv_e->va ^ vaddr))) {
+               if ((popcnt1((uintptr_t)pv_e->pmap ^ (uintptr_t)pmap) && PVE_VA(pv_e) == vaddr) ||
+                   (pv_e->pmap == pmap && popcnt1(PVE_VA(pv_e) ^ vaddr))) {

                        pv_e->pmap = pmap;
                        pv_e->pmap = pmap;

-                       pv_e->va = vaddr;
+                       if (pv_e == pv_h) {
+                               pv_h->va_and_flags = vaddr | pve_flags;
+                       } else {
+                               pv_e->va_and_flags = vaddr;
+                       }

                        suppress_reason = PV_BITFLIP;
                        action = PMAP_ACTION_RETRY;
                        goto pmap_cpc_exit;
                        suppress_reason = PV_BITFLIP;
                        action = PMAP_ACTION_RETRY;
                        goto pmap_cpc_exit;


@@ -688,7 +704,7 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t *
                ppnum_t npn = cpn ^ (ppnum_t) (1ULL << bitdex);
                if (IS_MANAGED_PAGE(npn)) {
                        pv_rooted_entry_t npv_h = pai_to_pvh(ppn_to_pai(npn));
                ppnum_t npn = cpn ^ (ppnum_t) (1ULL << bitdex);
                if (IS_MANAGED_PAGE(npn)) {
                        pv_rooted_entry_t npv_h = pai_to_pvh(ppn_to_pai(npn));

-                       if (npv_h->va == vaddr && npv_h->pmap == pmap) {
+                       if (PVE_VA(npv_h) == vaddr && npv_h->pmap == pmap) {

                                suppress_reason = PTE_BITFLIP;
                                suppress_ppn = npn;
                                action = PMAP_ACTION_RETRY_RELOCK;
                                suppress_reason = PTE_BITFLIP;
                                suppress_ppn = npn;
                                action = PMAP_ACTION_RETRY_RELOCK;


@@ -704,9 +720,11 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t *
                goto pmap_cpc_exit;
        }
 
                goto pmap_cpc_exit;
        }
 

-       /* Check for malformed/inconsistent entries */
-
-       if ((cpte & (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU | INTEL_PTE_PTA)) ==  (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU)) {
+       /*
+        * Check for malformed/inconsistent entries.
+        * The first check here isn't useful for EPT PTEs because INTEL_EPT_NCACHE == 0
+        */
+       if (!is_ept && ((cpte & (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU | INTEL_PTE_PTA)) ==  (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU))) {

                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_INVALID_CACHEABILITY;
        }
                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_INVALID_CACHEABILITY;
        }


@@ -714,7 +732,7 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t *
                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_RSVD;
        }
                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_RSVD;
        }

-       else if ((pmap != kernel_pmap) && ((cpte & INTEL_PTE_USER) == 0)) {
+       else if ((pmap != kernel_pmap) && (!is_ept) && ((cpte & INTEL_PTE_USER) == 0)) {

                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_SUPERVISOR;
        }
                action = PMAP_ACTION_IGNORE;
                suppress_reason = PTE_SUPERVISOR;
        }


@@ -775,7 +793,7 @@ pmap_pv_remove_retry:
                        goto pmap_pv_remove_retry;
        }
 
                        goto pmap_pv_remove_retry;
        }
 

-       if (pv_h->va == vaddr && pv_h->pmap == pmap) {
+       if (PVE_VA(pv_h) == vaddr && pv_h->pmap == pmap) {

                /*
                 * Header is the pv_rooted_entry.
                 * We can't free that. If there is a queued
                /*
                 * Header is the pv_rooted_entry.
                 * We can't free that. If there is a queued


@@ -785,12 +803,14 @@ pmap_pv_remove_retry:
                 */
                pvh_e = (pv_hashed_entry_t) queue_next(&pv_h->qlink);
                if (pv_h != (pv_rooted_entry_t) pvh_e) {
                 */
                pvh_e = (pv_hashed_entry_t) queue_next(&pv_h->qlink);
                if (pv_h != (pv_rooted_entry_t) pvh_e) {

+                       vm_map_offset_t pve_flags;
+

                        /*
                         * Entry queued to root, remove this from hash
                         * and install as new root.
                         */
                        CHK_NPVHASH();
                        /*
                         * Entry queued to root, remove this from hash
                         * and install as new root.
                         */
                        CHK_NPVHASH();

-                       pvhash_idx = pvhashidx(pvh_e->pmap, pvh_e->va);
+                       pvhash_idx = pvhashidx(pvh_e->pmap, PVE_VA(pvh_e));

                        LOCK_PV_HASH(pvhash_idx);
                        remque(&pvh_e->qlink);
                        pprevh = pvhash(pvhash_idx);
                        LOCK_PV_HASH(pvhash_idx);
                        remque(&pvh_e->qlink);
                        pprevh = pvhash(pvhash_idx);


@@ -802,7 +822,9 @@ pmap_pv_remove_retry:
                        pmap_pvh_unlink(pvh_e);
                        UNLOCK_PV_HASH(pvhash_idx);
                        pv_h->pmap = pvh_e->pmap;
                        pmap_pvh_unlink(pvh_e);
                        UNLOCK_PV_HASH(pvhash_idx);
                        pv_h->pmap = pvh_e->pmap;

-                       pv_h->va = pvh_e->va;   /* dispose of pvh_e */
+                       pve_flags = PVE_FLAGS(pv_h);
+                       pv_h->va_and_flags = PVE_VA(pvh_e) | pve_flags;
+                       /* dispose of pvh_e */

                } else {
                        /* none queued after rooted */
                        pv_h->pmap = PMAP_NULL;
                } else {
                        /* none queued after rooted */
                        pv_h->pmap = PMAP_NULL;


@@ -827,7 +849,7 @@ pmap_pv_remove_retry:
                while (PV_HASHED_ENTRY_NULL != pvh_e) {
                        pv_cnt++;
                        if (pvh_e->pmap == pmap &&
                while (PV_HASHED_ENTRY_NULL != pvh_e) {
                        pv_cnt++;
                        if (pvh_e->pmap == pmap &&

-                           pvh_e->va == vaddr &&
+                           PVE_VA(pvh_e) == vaddr &&

                            pvh_e->ppn == ppn)
                                break;
                        pprevh = &pvh_e->nexth;
                            pvh_e->ppn == ppn)
                                break;
                        pprevh = &pvh_e->nexth;


@@ -838,7 +860,7 @@ pmap_pv_remove_retry:
                        pmap_pagetable_corruption_action_t pac = pmap_classify_pagetable_corruption(pmap, vaddr, ppnp, pte, ROOT_PRESENT);
 
                        if (pac == PMAP_ACTION_ASSERT)
                        pmap_pagetable_corruption_action_t pac = pmap_classify_pagetable_corruption(pmap, vaddr, ppnp, pte, ROOT_PRESENT);
 
                        if (pac == PMAP_ACTION_ASSERT)

-                               panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, pv_h->va);
+                               panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, PVE_VA(pv_h));

                        else {
                                UNLOCK_PV_HASH(pvhash_idx);
                                if (pac == PMAP_ACTION_RETRY_RELOCK) {
                        else {
                                UNLOCK_PV_HASH(pvhash_idx);
                                if (pac == PMAP_ACTION_RETRY_RELOCK) {


@@ -871,29 +893,13 @@ extern int        pt_fake_zone_index;
 static inline void
 PMAP_ZINFO_PALLOC(pmap_t pmap, vm_size_t bytes)
 {
 static inline void
 PMAP_ZINFO_PALLOC(pmap_t pmap, vm_size_t bytes)
 {

-       thread_t thr = current_thread();
-       task_t task;
-       zinfo_usage_t zinfo;
-

        pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes);
        pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes);

-
-       if (pt_fake_zone_index != -1 && 
-           (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
-               OSAddAtomic64(bytes, (int64_t *)&zinfo[pt_fake_zone_index].alloc);

 }
 
 static inline void
 PMAP_ZINFO_PFREE(pmap_t pmap, vm_size_t bytes)
 {
 }
 
 static inline void
 PMAP_ZINFO_PFREE(pmap_t pmap, vm_size_t bytes)
 {

-       thread_t thr = current_thread();
-       task_t task;
-       zinfo_usage_t zinfo;
-

        pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes);
        pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes);

-
-       if (pt_fake_zone_index != -1 && 
-           (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
-               OSAddAtomic64(bytes, (int64_t *)&zinfo[pt_fake_zone_index].free);

 }
 
 static inline void
 }
 
 static inline void


@@ -995,7 +1001,7 @@ pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr)
                return (NULL);
        }
 
                return (NULL);
        }
 

-#if    PMAP_ASSERT
+#if    DEBUG

        return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_cr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]);
 #else
        return &pmap->pm_pml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)];
        return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_cr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]);
 #else
        return &pmap->pm_pml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)];


@@ -1010,9 +1016,12 @@ pmap64_pdpt(pmap_t pmap, vm_map_offset_t vaddr)
 {
        pml4_entry_t    newpf;
        pml4_entry_t    *pml4;
 {
        pml4_entry_t    newpf;
        pml4_entry_t    *pml4;

+       boolean_t       is_ept;

 
        pml4 = pmap64_pml4(pmap, vaddr);
 
        pml4 = pmap64_pml4(pmap, vaddr);

-       if (pml4 && ((*pml4 & INTEL_PTE_VALID))) {
+       is_ept = is_ept_pmap(pmap);
+
+       if (pml4 && (*pml4 & PTE_VALID_MASK(is_ept))) {

                newpf = *pml4 & PG_FRAME;
                return &((pdpt_entry_t *) PHYSMAP_PTOV(newpf))
                        [(vaddr >> PDPTSHIFT) & (NPDPTPG-1)];
                newpf = *pml4 & PG_FRAME;
                return &((pdpt_entry_t *) PHYSMAP_PTOV(newpf))
                        [(vaddr >> PDPTSHIFT) & (NPDPTPG-1)];


@@ -1027,10 +1036,12 @@ pmap64_pde(pmap_t pmap, vm_map_offset_t vaddr)
 {
        pdpt_entry_t    newpf;
        pdpt_entry_t    *pdpt;
 {
        pdpt_entry_t    newpf;
        pdpt_entry_t    *pdpt;

+       boolean_t       is_ept;

 
        pdpt = pmap64_pdpt(pmap, vaddr);
 
        pdpt = pmap64_pdpt(pmap, vaddr);

+       is_ept = is_ept_pmap(pmap);

 
 

-       if (pdpt && ((*pdpt & INTEL_PTE_VALID))) {
+       if (pdpt && (*pdpt & PTE_VALID_MASK(is_ept))) {

                newpf = *pdpt & PG_FRAME;
                return &((pd_entry_t *) PHYSMAP_PTOV(newpf))
                        [(vaddr >> PDSHIFT) & (NPDPG-1)];
                newpf = *pdpt & PG_FRAME;
                return &((pd_entry_t *) PHYSMAP_PTOV(newpf))
                        [(vaddr >> PDSHIFT) & (NPDPG-1)];


@@ -1060,12 +1071,15 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
 {
        pd_entry_t      *pde;
        pd_entry_t      newpf;
 {
        pd_entry_t      *pde;
        pd_entry_t      newpf;

+       boolean_t       is_ept;

 
        assert(pmap);
        pde = pmap64_pde(pmap, vaddr);
 
 
        assert(pmap);
        pde = pmap64_pde(pmap, vaddr);
 

-       if (pde && ((*pde & INTEL_PTE_VALID))) {
-               if (*pde & INTEL_PTE_PS) 
+       is_ept = is_ept_pmap(pmap);
+
+       if (pde && (*pde & PTE_VALID_MASK(is_ept))) {
+               if (*pde & PTE_PS)

                        return pde;
                newpf = *pde & PG_FRAME;
                return &((pt_entry_t *)PHYSMAP_PTOV(newpf))
                        return pde;
                newpf = *pde & PG_FRAME;
                return &((pt_entry_t *)PHYSMAP_PTOV(newpf))