xnu-4570.31.3.tar.gz

[apple/xnu.git] / osfmk / i386 / pmap_x86_common.c

diff --git a/osfmk/i386/pmap_x86_common.c b/osfmk/i386/pmap_x86_common.c
index 9841a0754f64cb741ad22f023d5f554a1b53c4e2..b66630233d686a24a621d04f9c05173c34c10227 100644 (file)
--- a/osfmk/i386/pmap_x86_common.c
+++ b/osfmk/i386/pmap_x86_common.c
@@ -30,6 +30,7 @@
 
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 
 #include <vm/pmap.h>
 #include <vm/vm_map.h>

+#include <vm/vm_kern.h>

 #include <kern/ledger.h>
 #include <i386/pmap_internal.h>
 
 #include <kern/ledger.h>
 #include <i386/pmap_internal.h>
 


@@ -111,8 +112,8 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t
                panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);
 
        PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
                panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);
 
        PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,

-       (uintptr_t) grand, (uintptr_t) subord,
-           (uintptr_t) (va_start>>32), (uintptr_t) va_start, 0);
+                  VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
+                  VM_KERNEL_ADDRHIDE(va_start));

 
        nvaddr = (vm_map_offset_t)nstart;
        num_pde = size >> PDESHIFT;
 
        nvaddr = (vm_map_offset_t)nstart;
        num_pde = size >> PDESHIFT;


@@ -178,11 +179,10 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t
                        i += (uint32_t) NPDEPG;
                }
                else {
                        i += (uint32_t) NPDEPG;
                }
                else {

-                       npde = pmap_pde(subord, nstart);
+                       npde = pmap_pde(subord, vaddr);

                        if (npde == 0)
                        if (npde == 0)

-                               panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
+                               panic("pmap_nest: no npde, subord %p vaddr 0x%llx", subord, vaddr);

                        tpde = *npde;
                        tpde = *npde;

-                       nstart += NBPDE;

                        pde = pmap_pde(grand, vaddr);
                        if ((0 == pde) && cpu_64bit) {
                                PMAP_UNLOCK(grand);
                        pde = pmap_pde(grand, vaddr);
                        if ((0 == pde) && cpu_64bit) {
                                PMAP_UNLOCK(grand);


@@ -201,7 +201,7 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t
 
        PMAP_UNLOCK(grand);
 
 
        PMAP_UNLOCK(grand);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, KERN_SUCCESS);

 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }


@@ -216,7 +216,6 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t
  */
 
 kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
  */
 
 kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {

-                       

        pd_entry_t *pde;
        unsigned int i;
        uint64_t num_pde;
        pd_entry_t *pde;
        unsigned int i;
        uint64_t num_pde;


@@ -224,8 +223,7 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
        uint64_t npdpt = PMAP_INVALID_PDPTNUM;
 
        PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
        uint64_t npdpt = PMAP_INVALID_PDPTNUM;
 
        PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,

-           (uintptr_t) grand, 
-           (uintptr_t) (vaddr>>32), (uintptr_t) vaddr, 0, 0);
+                  VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));

 
        if ((size & (pmap_nesting_size_min-1)) ||
            (vaddr & (pmap_nesting_size_min-1))) {
 
        if ((size & (pmap_nesting_size_min-1)) ||
            (vaddr & (pmap_nesting_size_min-1))) {


@@ -267,8 +265,8 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
        PMAP_UPDATE_TLBS(grand, va_start, va_end);
 
        PMAP_UNLOCK(grand);
        PMAP_UPDATE_TLBS(grand, va_start, va_end);
 
        PMAP_UNLOCK(grand);

-               
-       PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+       PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, KERN_SUCCESS);

 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }


@@ -405,7 +403,7 @@ pmap_update_cache_attributes_locked(ppnum_t pn, unsigned attributes) {
 
                do {
                        pmap = pv_e->pmap;
 
                do {
                        pmap = pv_e->pmap;

-                       vaddr = pv_e->va;
+                       vaddr = PVE_VA(pv_e);

                        ptep = pmap_pte(pmap, vaddr);
                        
                        if (0 == ptep)
                        ptep = pmap_pte(pmap, vaddr);
                        
                        if (0 == ptep)


@@ -452,9 +450,9 @@ void x86_filter_TLB_coherency_interrupts(boolean_t dofilter) {
  *     insert this page into the given map NOW.
  */
 
  *     insert this page into the given map NOW.
  */
 

-void
+kern_return_t

 pmap_enter(
 pmap_enter(

-       register pmap_t         pmap,
+       pmap_t          pmap,

        vm_map_offset_t         vaddr,
        ppnum_t                 pn,
        vm_prot_t               prot,
        vm_map_offset_t         vaddr,
        ppnum_t                 pn,
        vm_prot_t               prot,


@@ -462,13 +460,13 @@ pmap_enter(
        unsigned int            flags,
        boolean_t               wired)
 {
        unsigned int            flags,
        boolean_t               wired)
 {

-       (void) pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL);
+       return pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL);

 }
 
 
 kern_return_t
 pmap_enter_options(
 }
 
 
 kern_return_t
 pmap_enter_options(

-       register pmap_t         pmap,
+       pmap_t          pmap,

        vm_map_offset_t         vaddr,
        ppnum_t                 pn,
        vm_prot_t               prot,
        vm_map_offset_t         vaddr,
        ppnum_t                 pn,
        vm_prot_t               prot,


@@ -495,8 +493,11 @@ pmap_enter_options(
        vm_object_t             delpage_pm_obj = NULL;
        uint64_t                delpage_pde_index = 0;
        pt_entry_t              old_pte;
        vm_object_t             delpage_pm_obj = NULL;
        uint64_t                delpage_pde_index = 0;
        pt_entry_t              old_pte;

-       kern_return_t           kr_expand;
+       kern_return_t           kr;

        boolean_t               is_ept;
        boolean_t               is_ept;

+       boolean_t               is_altacct;
+
+       kr = KERN_FAILURE;

 
        pmap_intr_assert();
 
 
        pmap_intr_assert();
 


@@ -514,9 +515,8 @@ pmap_enter_options(
                return KERN_INVALID_ARGUMENT;
 
        PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
                return KERN_INVALID_ARGUMENT;
 
        PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,

-           pmap,
-           (uint32_t) (vaddr >> 32), (uint32_t) vaddr,
-           pn, prot);
+                  VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(vaddr), pn,
+                  prot);

 
        if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
                set_NX = FALSE;
 
        if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
                set_NX = FALSE;


@@ -546,13 +546,13 @@ Retry:
         *      pmap is always expanded to include enough hardware
         *      pages to map one VM page.
         */
         *      pmap is always expanded to include enough hardware
         *      pages to map one VM page.
         */

-        if(superpage) {
+        if (superpage) {

                while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) {
                        /* need room for another pde entry */
                        PMAP_UNLOCK(pmap);
                while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) {
                        /* need room for another pde entry */
                        PMAP_UNLOCK(pmap);

-                       kr_expand = pmap_expand_pdpt(pmap, vaddr, options);
-                       if (kr_expand != KERN_SUCCESS)
-                               return kr_expand;
+                       kr = pmap_expand_pdpt(pmap, vaddr, options);
+                       if (kr != KERN_SUCCESS)
+                               goto done;

                        PMAP_LOCK(pmap);
                }
        } else {
                        PMAP_LOCK(pmap);
                }
        } else {


@@ -562,15 +562,16 @@ Retry:
                         * going to grow pde level page(s)
                         */
                        PMAP_UNLOCK(pmap);
                         * going to grow pde level page(s)
                         */
                        PMAP_UNLOCK(pmap);

-                       kr_expand = pmap_expand(pmap, vaddr, options);
-                       if (kr_expand != KERN_SUCCESS)
-                               return kr_expand;
+                       kr = pmap_expand(pmap, vaddr, options);
+                       if (kr != KERN_SUCCESS)
+                               goto done;

                        PMAP_LOCK(pmap);
                }
        }
        if (options & PMAP_EXPAND_OPTIONS_NOENTER) {
                PMAP_UNLOCK(pmap);
                        PMAP_LOCK(pmap);
                }
        }
        if (options & PMAP_EXPAND_OPTIONS_NOENTER) {
                PMAP_UNLOCK(pmap);

-               return KERN_SUCCESS;
+               kr = KERN_SUCCESS;
+               goto done;

        }
 
        if (superpage && *pte && !(*pte & PTE_PS)) {
        }
 
        if (superpage && *pte && !(*pte & PTE_PS)) {


@@ -590,9 +591,27 @@ Retry:
        old_pa_locked = FALSE;
 
        if (old_pa == 0 &&
        old_pa_locked = FALSE;
 
        if (old_pa == 0 &&

-           (*pte & PTE_COMPRESSED)) {
+           PTE_IS_COMPRESSED(*pte)) {
+               /*
+                * "pmap" should be locked at this point, so this should
+                * not race with another pmap_enter() or pmap_remove_range().
+                */
+               assert(pmap != kernel_pmap);
+

                /* one less "compressed" */
                OSAddAtomic64(-1, &pmap->stats.compressed);
                /* one less "compressed" */
                OSAddAtomic64(-1, &pmap->stats.compressed);

+               pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
+                                 PAGE_SIZE);
+               if (*pte & PTE_COMPRESSED_ALT) {
+                       pmap_ledger_debit(
+                               pmap,
+                               task_ledgers.alternate_accounting_compressed,
+                               PAGE_SIZE);
+               } else {
+                       /* was part of the footprint */
+                       pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
+                                         PAGE_SIZE);
+               }

                /* marker will be cleared below */
        }
 
                /* marker will be cleared below */
        }
 


@@ -690,7 +709,7 @@ Retry:
                /* Determine delta, PV locked */
                need_tlbflush =
                    ((old_attributes ^ template) != PTE_WIRED);
                /* Determine delta, PV locked */
                need_tlbflush =
                    ((old_attributes ^ template) != PTE_WIRED);

-               
+

                if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) {
                        if ((old_attributes ^ template) == PTE_WRITE(is_ept))
                                need_tlbflush = FALSE;
                if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) {
                        if ((old_attributes ^ template) == PTE_WRITE(is_ept))
                                need_tlbflush = FALSE;


@@ -733,6 +752,7 @@ dont_update_pte:
         */
 
        if (old_pa != (pmap_paddr_t) 0) {
         */
 
        if (old_pa != (pmap_paddr_t) 0) {

+               boolean_t       was_altacct = FALSE;

 
                /*
                 *      Don't do anything to pages outside valid memory here.
 
                /*
                 *      Don't do anything to pages outside valid memory here.


@@ -750,23 +770,60 @@ dont_update_pte:
                /* completely invalidate the PTE */
                pmap_store_pte(pte, 0);
 
                /* completely invalidate the PTE */
                pmap_store_pte(pte, 0);
 

+               if (IS_MANAGED_PAGE(pai)) {
+                       /*
+                        *      Remove the mapping from the pvlist for
+                        *      this physical page.
+                        *      We'll end up with either a rooted pv or a
+                        *      hashed pv
+                        */
+                       pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte, &was_altacct);
+               }
+

                if (IS_MANAGED_PAGE(pai)) {
                        pmap_assert(old_pa_locked == TRUE);
                        pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
                if (IS_MANAGED_PAGE(pai)) {
                        pmap_assert(old_pa_locked == TRUE);
                        pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);

-                       pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);                        

                        assert(pmap->stats.resident_count >= 1);
                        OSAddAtomic(-1, &pmap->stats.resident_count);
                        if (pmap != kernel_pmap) {
                        assert(pmap->stats.resident_count >= 1);
                        OSAddAtomic(-1, &pmap->stats.resident_count);
                        if (pmap != kernel_pmap) {

+                               /* update pmap stats */

                                if (IS_REUSABLE_PAGE(pai)) {
                                if (IS_REUSABLE_PAGE(pai)) {

-                                       assert(pmap->stats.reusable > 0);
+                                       PMAP_STATS_ASSERTF(
+                                               (pmap->stats.reusable > 0,
+                                                "reusable %d",
+                                                pmap->stats.reusable));

                                        OSAddAtomic(-1, &pmap->stats.reusable);
                                } else if (IS_INTERNAL_PAGE(pai)) {
                                        OSAddAtomic(-1, &pmap->stats.reusable);
                                } else if (IS_INTERNAL_PAGE(pai)) {

-                                       assert(pmap->stats.internal > 0);
+                                       PMAP_STATS_ASSERTF(
+                                               (pmap->stats.internal > 0,
+                                                "internal %d",
+                                                pmap->stats.internal));

                                        OSAddAtomic(-1, &pmap->stats.internal);
                                } else {
                                        OSAddAtomic(-1, &pmap->stats.internal);
                                } else {

-                                       assert(pmap->stats.external > 0);
+                                       PMAP_STATS_ASSERTF(
+                                               (pmap->stats.external > 0,
+                                                "external %d",
+                                                pmap->stats.external));

                                        OSAddAtomic(-1, &pmap->stats.external);
                                }
                                        OSAddAtomic(-1, &pmap->stats.external);
                                }

+
+                               /* update ledgers */
+                               if (was_altacct) {
+                                       assert(IS_INTERNAL_PAGE(pai));
+                                       pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+                                       pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+                               } else if (IS_REUSABLE_PAGE(pai)) {
+                                       assert(!was_altacct);
+                                       assert(IS_INTERNAL_PAGE(pai));
+                                       /* was already not in phys_footprint */
+                               } else if (IS_INTERNAL_PAGE(pai)) {
+                                       assert(!was_altacct);
+                                       assert(!IS_REUSABLE_PAGE(pai));
+                                       pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+                                       pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+                               } else {
+                                       /* not an internal page */
+                               }

                        }
                        if (iswired(*pte)) {
                                assert(pmap->stats.wired_count >= 1);
                        }
                        if (iswired(*pte)) {
                                assert(pmap->stats.wired_count >= 1);


@@ -781,14 +838,6 @@ dont_update_pte:
                                pmap_phys_attributes[pai] |= ept_refmod_to_physmap(oattr);
                        }
 
                                pmap_phys_attributes[pai] |= ept_refmod_to_physmap(oattr);
                        }
 

-                       /*
-                        *      Remove the mapping from the pvlist for
-                        *      this physical page.
-                        *      We'll end up with either a rooted pv or a
-                        *      hashed pv
-                        */
-                       pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte);
-

                } else {
 
                        /*
                } else {
 
                        /*


@@ -833,7 +882,7 @@ dont_update_pte:
                        /*
                         *      No mappings yet, use rooted pv
                         */
                        /*
                         *      No mappings yet, use rooted pv
                         */

-                       pv_h->va = vaddr;
+                       pv_h->va_and_flags = vaddr;

                        pv_h->pmap = pmap;
                        queue_init(&pv_h->qlink);
 
                        pv_h->pmap = pmap;
                        queue_init(&pv_h->qlink);
 


@@ -847,6 +896,14 @@ dont_update_pte:
                        } else {
                                pmap_phys_attributes[pai] &= ~PHYS_REUSABLE;
                        }
                        } else {
                                pmap_phys_attributes[pai] &= ~PHYS_REUSABLE;
                        }

+                       if ((options & PMAP_OPTIONS_ALT_ACCT) &&
+                           IS_INTERNAL_PAGE(pai)) {
+                               pv_h->va_and_flags |= PVE_IS_ALTACCT;
+                               is_altacct = TRUE;
+                       } else {
+                               pv_h->va_and_flags &= ~PVE_IS_ALTACCT;
+                               is_altacct = FALSE;
+                       }

                } else {
                        /*
                         *      Add new pv_hashed_entry after header.
                } else {
                        /*
                         *      Add new pv_hashed_entry after header.


@@ -877,13 +934,21 @@ dont_update_pte:
                                        }
                                }
                        }
                                        }
                                }
                        }

-                       
+

                        if (PV_HASHED_ENTRY_NULL == pvh_e)
                                panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings");
 
                        if (PV_HASHED_ENTRY_NULL == pvh_e)
                                panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings");
 

-                       pvh_e->va = vaddr;
+                       pvh_e->va_and_flags = vaddr;

                        pvh_e->pmap = pmap;
                        pvh_e->ppn = pn;
                        pvh_e->pmap = pmap;
                        pvh_e->ppn = pn;

+                       if ((options & PMAP_OPTIONS_ALT_ACCT) &&
+                           IS_INTERNAL_PAGE(pai)) {
+                               pvh_e->va_and_flags |= PVE_IS_ALTACCT;
+                               is_altacct = TRUE;
+                       } else {
+                               pvh_e->va_and_flags &= ~PVE_IS_ALTACCT;
+                               is_altacct = FALSE;
+                       }

                        pv_hash_add(pvh_e, pv_h);
 
                        /*
                        pv_hash_add(pvh_e, pv_h);
 
                        /*


@@ -897,12 +962,12 @@ dont_update_pte:
                 * for 'managed memory'
                 */
                pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
                 * for 'managed memory'
                 */
                pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);

-               pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);               

                OSAddAtomic(+1,  &pmap->stats.resident_count);
                if (pmap->stats.resident_count > pmap->stats.resident_max) {
                        pmap->stats.resident_max = pmap->stats.resident_count;
                }
                if (pmap != kernel_pmap) {
                OSAddAtomic(+1,  &pmap->stats.resident_count);
                if (pmap->stats.resident_count > pmap->stats.resident_max) {
                        pmap->stats.resident_max = pmap->stats.resident_count;
                }
                if (pmap != kernel_pmap) {

+                       /* update pmap stats */

                        if (IS_REUSABLE_PAGE(pai)) {
                                OSAddAtomic(+1, &pmap->stats.reusable);
                                PMAP_STATS_PEAK(pmap->stats.reusable);
                        if (IS_REUSABLE_PAGE(pai)) {
                                OSAddAtomic(+1, &pmap->stats.reusable);
                                PMAP_STATS_PEAK(pmap->stats.reusable);


@@ -913,13 +978,33 @@ dont_update_pte:
                                OSAddAtomic(+1, &pmap->stats.external);
                                PMAP_STATS_PEAK(pmap->stats.external);
                        }
                                OSAddAtomic(+1, &pmap->stats.external);
                                PMAP_STATS_PEAK(pmap->stats.external);
                        }

+
+                       /* update ledgers */
+                       if (is_altacct) {
+                               /* internal but also alternate accounting */
+                               assert(IS_INTERNAL_PAGE(pai));
+                               pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
+                               pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+                               /* alternate accounting, so not in footprint */
+                       } else if (IS_REUSABLE_PAGE(pai)) {
+                               assert(!is_altacct);
+                               assert(IS_INTERNAL_PAGE(pai));
+                               /* internal but reusable: not in footprint */
+                       } else if (IS_INTERNAL_PAGE(pai)) {
+                               assert(!is_altacct);
+                               assert(!IS_REUSABLE_PAGE(pai));
+                               /* internal: add to footprint */
+                               pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
+                               pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+                       } else {
+                               /* not internal: not in footprint */
+                       }

                }
        } else if (last_managed_page == 0) {
                /* Account for early mappings created before "managed pages"
                 * are determined. Consider consulting the available DRAM map.
                 */
                pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
                }
        } else if (last_managed_page == 0) {
                /* Account for early mappings created before "managed pages"
                 * are determined. Consider consulting the available DRAM map.
                 */
                pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);

-               pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);               

                OSAddAtomic(+1,  &pmap->stats.resident_count);
                if (pmap != kernel_pmap) {
 #if 00
                OSAddAtomic(+1,  &pmap->stats.resident_count);
                if (pmap != kernel_pmap) {
 #if 00


@@ -1030,8 +1115,10 @@ Done:
                PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
        }
 
                PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
        }
 

-       PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, 0, 0, 0, 0, 0);
-       return KERN_SUCCESS;
+       kr = KERN_SUCCESS;
+done:
+       PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
+       return kr;

 }
 
 /*
 }
 
 /*


@@ -1071,22 +1158,28 @@ pmap_remove_range_options(
        pv_hashed_entry_t       pvh_e;
        int                     pvh_cnt = 0;
        int                     num_removed, num_unwired, num_found, num_invalid;
        pv_hashed_entry_t       pvh_e;
        int                     pvh_cnt = 0;
        int                     num_removed, num_unwired, num_found, num_invalid;

-       int                     num_device, num_external, num_internal, num_reusable;
-       uint64_t                num_compressed;
+       int                     stats_external, stats_internal, stats_reusable;
+       uint64_t                stats_compressed;
+       int                     ledgers_internal, ledgers_alt_internal;
+       uint64_t                ledgers_compressed, ledgers_alt_compressed;

        ppnum_t                 pai;
        pmap_paddr_t            pa;
        vm_map_offset_t         vaddr;
        boolean_t               is_ept = is_ept_pmap(pmap);
        ppnum_t                 pai;
        pmap_paddr_t            pa;
        vm_map_offset_t         vaddr;
        boolean_t               is_ept = is_ept_pmap(pmap);

+       boolean_t               was_altacct;

 
        num_removed = 0;
        num_unwired = 0;
        num_found   = 0;
        num_invalid = 0;
 
        num_removed = 0;
        num_unwired = 0;
        num_found   = 0;
        num_invalid = 0;

-       num_device  = 0;
-       num_external = 0;
-       num_internal = 0;
-       num_reusable = 0;
-       num_compressed = 0;
+       stats_external = 0;
+       stats_internal = 0;
+       stats_reusable = 0;
+       stats_compressed = 0;
+       ledgers_internal = 0;
+       ledgers_compressed = 0;
+       ledgers_alt_internal = 0;
+       ledgers_alt_compressed = 0;

        /* invalidate the PTEs first to "freeze" them */
        for (cpte = spte, vaddr = start_vaddr;
             cpte < epte;
        /* invalidate the PTEs first to "freeze" them */
        for (cpte = spte, vaddr = start_vaddr;
             cpte < epte;


@@ -1095,14 +1188,19 @@ pmap_remove_range_options(
 
                pa = pte_to_pa(p);
                if (pa == 0) {
 
                pa = pte_to_pa(p);
                if (pa == 0) {

-                       if (pmap != kernel_pmap &&
-                           (options & PMAP_OPTIONS_REMOVE) &&
-                           (p & PTE_COMPRESSED)) {
-                               /* one less "compressed" */
-                               num_compressed++;
-                               /* clear marker */
+                       if ((options & PMAP_OPTIONS_REMOVE) &&
+                           (PTE_IS_COMPRESSED(p))) {
+                               assert(pmap != kernel_pmap);
+                               /* one less "compressed"... */
+                               stats_compressed++;
+                               ledgers_compressed++;
+                               if (p & PTE_COMPRESSED_ALT) {
+                                       /* ... but it used to be "ALTACCT" */
+                                       ledgers_alt_compressed++;
+                               }
+                               /* clear marker(s) */

                                /* XXX probably does not need to be atomic! */
                                /* XXX probably does not need to be atomic! */

-                               pmap_update_pte(cpte, PTE_COMPRESSED, 0);
+                               pmap_update_pte(cpte, INTEL_PTE_COMPRESSED_MASK, 0);

                        }
                        continue;
                }
                        }
                        continue;
                }


@@ -1119,7 +1217,6 @@ pmap_remove_range_options(
                         *      Just remove the mappings.
                         */
                        pmap_store_pte(cpte, 0);
                         *      Just remove the mappings.
                         */
                        pmap_store_pte(cpte, 0);

-                       num_device++;

                        continue;
                }
 
                        continue;
                }
 


@@ -1144,8 +1241,27 @@ pmap_remove_range_options(
             cpte++, vaddr += PAGE_SIZE_64) {
 
                pa = pte_to_pa(*cpte);
             cpte++, vaddr += PAGE_SIZE_64) {
 
                pa = pte_to_pa(*cpte);

-               if (pa == 0)
+               if (pa == 0) {
+               check_pte_for_compressed_marker:
+                       /*
+                        * This PTE could have been replaced with a
+                        * "compressed" marker after our first "freeze"
+                        * loop above, so check again.
+                        */
+                       if ((options & PMAP_OPTIONS_REMOVE) &&
+                           (PTE_IS_COMPRESSED(*cpte))) {
+                               assert(pmap != kernel_pmap);
+                               /* one less "compressed"... */
+                               stats_compressed++;
+                               ledgers_compressed++;
+                               if (*cpte & PTE_COMPRESSED_ALT) {
+                                       /* ... but it used to be "ALTACCT" */
+                                       ledgers_alt_compressed++;
+                               }
+                               pmap_store_pte(cpte, 0);
+                       }

                        continue;
                        continue;

+               }

 
                pai = pa_index(pa);
 
 
                pai = pa_index(pa);
 


@@ -1154,15 +1270,40 @@ pmap_remove_range_options(
                pa = pte_to_pa(*cpte);
                if (pa == 0) {
                        UNLOCK_PVH(pai);
                pa = pte_to_pa(*cpte);
                if (pa == 0) {
                        UNLOCK_PVH(pai);

-                       continue;
+                       goto check_pte_for_compressed_marker;

                }
                }

+
+               /*
+                * Remove the mapping from the pvlist for this physical page.
+                */
+               pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte, &was_altacct);
+

                num_removed++;
                num_removed++;

+               /* update pmap stats */

                if (IS_REUSABLE_PAGE(pai)) {
                if (IS_REUSABLE_PAGE(pai)) {

-                       num_reusable++;
+                       stats_reusable++;
+               } else if (IS_INTERNAL_PAGE(pai)) {
+                       stats_internal++;
+               } else {
+                       stats_external++;
+               }
+               /* update ledgers */
+               if (was_altacct) {
+                       /* internal and alternate accounting */
+                       assert(IS_INTERNAL_PAGE(pai));
+                       ledgers_internal++;
+                       ledgers_alt_internal++;
+               } else if (IS_REUSABLE_PAGE(pai)) {
+                       /* internal but reusable */
+                       assert(!was_altacct);
+                       assert(IS_INTERNAL_PAGE(pai));

                } else if (IS_INTERNAL_PAGE(pai)) {
                } else if (IS_INTERNAL_PAGE(pai)) {

-                       num_internal++;
+                       /* internal */
+                       assert(!was_altacct);
+                       assert(!IS_REUSABLE_PAGE(pai));
+                       ledgers_internal++;

                } else {
                } else {

-                       num_external++;
+                       /* not internal */

                }
 
                /*
                }
 
                /*


@@ -1170,13 +1311,13 @@ pmap_remove_range_options(
                 * nuke the entry in the page table
                 */
                /* remember reference and change */
                 * nuke the entry in the page table
                 */
                /* remember reference and change */

-               pmap_phys_attributes[pai] |=
-                       (char) (*cpte & (PHYS_MODIFIED | PHYS_REFERENCED));
-
-               /*
-                * Remove the mapping from the pvlist for this physical page.
-                */
-               pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte);
+               if (!is_ept) {
+                       pmap_phys_attributes[pai] |=
+                               *cpte & (PHYS_MODIFIED | PHYS_REFERENCED);
+               } else {
+                       pmap_phys_attributes[pai] |=
+                               ept_refmod_to_physmap((*cpte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+               }

 
                /* completely invalidate the PTE */
                pmap_store_pte(cpte, 0);
 
                /* completely invalidate the PTE */
                pmap_store_pte(cpte, 0);


@@ -1206,35 +1347,72 @@ update_counts:
                panic("pmap_remove_range: resident_count");
 #endif
        pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));
                panic("pmap_remove_range: resident_count");
 #endif
        pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));

-       pmap_ledger_debit(pmap, task_ledgers.phys_footprint, machine_ptob(num_removed));        
-       assert(pmap->stats.resident_count >= num_removed);
+       PMAP_STATS_ASSERTF((pmap->stats.resident_count >= num_removed,
+                           "pmap=%p num_removed=%d stats.resident_count=%d",
+                           pmap, num_removed, pmap->stats.resident_count));

        OSAddAtomic(-num_removed,  &pmap->stats.resident_count);
 
        if (pmap != kernel_pmap) {
        OSAddAtomic(-num_removed,  &pmap->stats.resident_count);
 
        if (pmap != kernel_pmap) {

-#if 00
-               assert(pmap->stats.device >= num_device);
-               if (num_device)
-                       OSAddAtomic(-num_device, &pmap->stats.device);
-#endif /* 00 */
-               assert(pmap->stats.external >= num_external);
-               if (num_external)
-                       OSAddAtomic(-num_external, &pmap->stats.external);
-               assert(pmap->stats.internal >= num_internal);
-               if (num_internal)
-                       OSAddAtomic(-num_internal, &pmap->stats.internal);
-               assert(pmap->stats.reusable >= num_reusable);
-               if (num_reusable)
-                       OSAddAtomic(-num_reusable, &pmap->stats.reusable);
-               assert(pmap->stats.compressed >= num_compressed);
-               if (num_compressed)
-                       OSAddAtomic64(-num_compressed, &pmap->stats.compressed);
+               PMAP_STATS_ASSERTF((pmap->stats.external >= stats_external,
+                                   "pmap=%p stats_external=%d stats.external=%d",
+                                   pmap, stats_external, pmap->stats.external));
+               PMAP_STATS_ASSERTF((pmap->stats.internal >= stats_internal,
+                                   "pmap=%p stats_internal=%d stats.internal=%d",
+                                   pmap, stats_internal, pmap->stats.internal));
+               PMAP_STATS_ASSERTF((pmap->stats.reusable >= stats_reusable,
+                                   "pmap=%p stats_reusable=%d stats.reusable=%d",
+                                   pmap, stats_reusable, pmap->stats.reusable));
+               PMAP_STATS_ASSERTF((pmap->stats.compressed >= stats_compressed,
+                                   "pmap=%p stats_compressed=%lld, stats.compressed=%lld",
+                                   pmap, stats_compressed, pmap->stats.compressed));
+
+               /* update pmap stats */
+               if (stats_external) {
+                       OSAddAtomic(-stats_external, &pmap->stats.external);
+               }
+               if (stats_internal) {
+                       OSAddAtomic(-stats_internal, &pmap->stats.internal);
+               }
+               if (stats_reusable)
+                       OSAddAtomic(-stats_reusable, &pmap->stats.reusable);
+               if (stats_compressed)
+                       OSAddAtomic64(-stats_compressed, &pmap->stats.compressed);
+               /* update ledgers */
+               if (ledgers_internal) {
+                       pmap_ledger_debit(pmap,
+                                         task_ledgers.internal,
+                                         machine_ptob(ledgers_internal));
+               }
+               if (ledgers_compressed) {
+                       pmap_ledger_debit(pmap,
+                                         task_ledgers.internal_compressed,
+                                         machine_ptob(ledgers_compressed));
+               }
+               if (ledgers_alt_internal) {
+                       pmap_ledger_debit(pmap,
+                                         task_ledgers.alternate_accounting,
+                                         machine_ptob(ledgers_alt_internal));
+               }
+               if (ledgers_alt_compressed) {
+                       pmap_ledger_debit(pmap,
+                                         task_ledgers.alternate_accounting_compressed,
+                                         machine_ptob(ledgers_alt_compressed));
+               }
+               pmap_ledger_debit(pmap,
+                                 task_ledgers.phys_footprint,
+                                 machine_ptob((ledgers_internal -
+                                               ledgers_alt_internal) +
+                                              (ledgers_compressed -
+                                               ledgers_alt_compressed)));

        }
 
 #if TESTING
        if (pmap->stats.wired_count < num_unwired)
                panic("pmap_remove_range: wired_count");
 #endif
        }
 
 #if TESTING
        if (pmap->stats.wired_count < num_unwired)
                panic("pmap_remove_range: wired_count");
 #endif

-       assert(pmap->stats.wired_count >= num_unwired);
+       PMAP_STATS_ASSERTF((pmap->stats.wired_count >= num_unwired,
+                           "pmap=%p num_unwired=%d stats.wired_count=%d",
+                           pmap, num_unwired, pmap->stats.wired_count));

        OSAddAtomic(-num_unwired,  &pmap->stats.wired_count);
        pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
 
        OSAddAtomic(-num_unwired,  &pmap->stats.wired_count);
        pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
 


@@ -1279,10 +1457,8 @@ pmap_remove_options(
        is_ept = is_ept_pmap(map);
 
        PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
        is_ept = is_ept_pmap(map);
 
        PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,

-                  map,
-                  (uint32_t) (s64 >> 32), s64,
-                  (uint32_t) (e64 >> 32), e64);
-
+                  VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(s64),
+                  VM_KERNEL_ADDRHIDE(e64));

 
        PMAP_LOCK(map);
 
 
        PMAP_LOCK(map);
 


@@ -1363,8 +1539,7 @@ pmap_remove_options(
 
        PMAP_UNLOCK(map);
 
 
        PMAP_UNLOCK(map);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END,
-                  map, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);

 
 }
 
 
 }
 


@@ -1417,8 +1592,8 @@ pmap_page_protect_options(
                 */
                return;
        }
                 */
                return;
        }

-       PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START,
-                  pn, prot, 0, 0, 0);
+
+       PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, pn, prot);

 
        /*
         * Determine the new protection.
 
        /*
         * Determine the new protection.


@@ -1461,7 +1636,7 @@ pmap_page_protect_options(
 
                pmap = pv_e->pmap;
                is_ept = is_ept_pmap(pmap);
 
                pmap = pv_e->pmap;
                is_ept = is_ept_pmap(pmap);

-               vaddr = pv_e->va;
+               vaddr = PVE_VA(pv_e);

                pte = pmap_pte(pmap, vaddr);
 
                pmap_assert2((pa_index(pte_to_pa(*pte)) == pn),
                pte = pmap_pte(pmap, vaddr);
 
                pmap_assert2((pa_index(pte_to_pa(*pte)) == pn),


@@ -1488,8 +1663,12 @@ pmap_page_protect_options(
                        if (pmap != kernel_pmap &&
                            (options & PMAP_OPTIONS_COMPRESSOR) &&
                            IS_INTERNAL_PAGE(pai)) {
                        if (pmap != kernel_pmap &&
                            (options & PMAP_OPTIONS_COMPRESSOR) &&
                            IS_INTERNAL_PAGE(pai)) {

-                               /* mark this PTE as having been "reclaimed" */
+                               assert(!PTE_IS_COMPRESSED(*pte));
+                               /* mark this PTE as having been "compressed" */

                                new_pte_value = PTE_COMPRESSED;
                                new_pte_value = PTE_COMPRESSED;

+                               if (IS_ALTACCT_PAGE(pai, pv_e)) {
+                                       new_pte_value |= PTE_COMPRESSED_ALT;
+                               }

                        } else {
                                new_pte_value = 0;
                        }
                        } else {
                                new_pte_value = 0;
                        }


@@ -1508,37 +1687,37 @@ pmap_page_protect_options(
                                pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0);
 
                                PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE);
                                pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0);
 
                                PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE);

+                               if (!is_ept) {
+                                       pmap_phys_attributes[pai] |=
+                                               *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
+                               } else {
+                                       pmap_phys_attributes[pai] |=
+                                               ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+                               }

                                if ((options &
                                     PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) &&
                                if ((options &
                                     PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) &&

-                                   ! (pmap_phys_attributes[pai] &
-                                      PHYS_MODIFIED) &&
-                                   (*pte & PHYS_MODIFIED)) {
+                                   IS_INTERNAL_PAGE(pai) &&
+                                   (pmap_phys_attributes[pai] &
+                                    PHYS_MODIFIED)) {

                                        /*
                                         * Page is actually "modified" and
                                         * will be compressed.  Start
                                         * accounting for it as "compressed".
                                         */
                                        /*
                                         * Page is actually "modified" and
                                         * will be compressed.  Start
                                         * accounting for it as "compressed".
                                         */

+                                       assert(!(options & PMAP_OPTIONS_COMPRESSOR));

                                        options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
                                        options |= PMAP_OPTIONS_COMPRESSOR;
                                        options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
                                        options |= PMAP_OPTIONS_COMPRESSOR;

-                                       new_pte_value = PTE_COMPRESSED;
-                               }
-                               if (!is_ept) {
-                                       pmap_phys_attributes[pai] |=
-                                               *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
-                               } else {
-                                       pmap_phys_attributes[pai] |=
-                                               ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+                                       assert(new_pte_value == 0);
+                                       if (pmap != kernel_pmap) {
+                                               new_pte_value = PTE_COMPRESSED;
+                                               if (IS_ALTACCT_PAGE(pai, pv_e)) {
+                                                       new_pte_value |= PTE_COMPRESSED_ALT;
+                                               }
+                                       }

                                }
                                pmap_store_pte(pte, new_pte_value);
                        }
 
                                }
                                pmap_store_pte(pte, new_pte_value);
                        }
 

-                       if (new_pte_value == PTE_COMPRESSED) {
-                               /* one more "compressed" page */
-                               OSAddAtomic64(+1, &pmap->stats.compressed);
-                               PMAP_STATS_PEAK(pmap->stats.compressed);
-                               pmap->stats.compressed_lifetime++;
-                       }
-

 #if TESTING
                        if (pmap->stats.resident_count < 1)
                                panic("pmap_page_protect: resident_count");
 #if TESTING
                        if (pmap->stats.resident_count < 1)
                                panic("pmap_page_protect: resident_count");


@@ -1546,17 +1725,15 @@ pmap_page_protect_options(
                        pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
                        assert(pmap->stats.resident_count >= 1);
                        OSAddAtomic(-1,  &pmap->stats.resident_count);
                        pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
                        assert(pmap->stats.resident_count >= 1);
                        OSAddAtomic(-1,  &pmap->stats.resident_count);

+
+                       /*
+                        * We only ever compress internal pages.
+                        */

                        if (options & PMAP_OPTIONS_COMPRESSOR) {
                        if (options & PMAP_OPTIONS_COMPRESSOR) {

-                               /*
-                                * This removal is only being done so we can send this page to
-                                * the compressor; therefore it mustn't affect total task footprint.
-                                */
-                               pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
-                       } else {
-                               pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+                               assert(IS_INTERNAL_PAGE(pai));

                        }
                        }

-

                        if (pmap != kernel_pmap) {
                        if (pmap != kernel_pmap) {

+                               /* update pmap stats */

                                if (IS_REUSABLE_PAGE(pai)) {
                                        assert(pmap->stats.reusable > 0);
                                        OSAddAtomic(-1, &pmap->stats.reusable);
                                if (IS_REUSABLE_PAGE(pai)) {
                                        assert(pmap->stats.reusable > 0);
                                        OSAddAtomic(-1, &pmap->stats.reusable);


@@ -1567,6 +1744,59 @@ pmap_page_protect_options(
                                        assert(pmap->stats.external > 0);
                                        OSAddAtomic(-1, &pmap->stats.external);
                                }
                                        assert(pmap->stats.external > 0);
                                        OSAddAtomic(-1, &pmap->stats.external);
                                }

+                               if ((options & PMAP_OPTIONS_COMPRESSOR) &&
+                                   IS_INTERNAL_PAGE(pai)) {
+                                       /* adjust "compressed" stats */
+                                       OSAddAtomic64(+1, &pmap->stats.compressed);
+                                       PMAP_STATS_PEAK(pmap->stats.compressed);
+                                       pmap->stats.compressed_lifetime++;
+                               }
+
+                               /* update ledgers */
+                               if (IS_ALTACCT_PAGE(pai, pv_e)) {
+                                       assert(IS_INTERNAL_PAGE(pai));
+                                       pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+                                       pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+                                       if (options & PMAP_OPTIONS_COMPRESSOR) {
+                                               pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+                                               pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE);
+                                       }
+                               } else if (IS_REUSABLE_PAGE(pai)) {
+                                       assert(!IS_ALTACCT_PAGE(pai, pv_e));
+                                       assert(IS_INTERNAL_PAGE(pai));
+                                       if (options & PMAP_OPTIONS_COMPRESSOR) {
+                                               pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+                                               /* was not in footprint, but is now */
+                                               pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+                                       }
+                               } else if (IS_INTERNAL_PAGE(pai)) {
+                                       assert(!IS_ALTACCT_PAGE(pai, pv_e));
+                                       assert(!IS_REUSABLE_PAGE(pai));
+                                       pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+                                       /*
+                                        * Update all stats related to physical
+                                        * footprint, which only deals with
+                                        * internal pages.
+                                        */
+                                       if (options & PMAP_OPTIONS_COMPRESSOR) {
+                                               /*
+                                                * This removal is only being
+                                                * done so we can send this page
+                                                * to the compressor;  therefore
+                                                * it mustn't affect total task
+                                                * footprint.
+                                                */
+                                               pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+                                       } else {
+                                               /*
+                                                * This internal page isn't
+                                                * going to the compressor,
+                                                * so adjust stats to keep 
+                                                * phys_footprint up to date.
+                                                */
+                                               pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+                                       }
+                               }

                        }
 
                        /*
                        }
 
                        /*


@@ -1621,7 +1851,7 @@ pmap_page_protect_options(
                if (pvh_e != (pv_hashed_entry_t) pv_h) {
                        pv_hash_remove(pvh_e);
                        pv_h->pmap = pvh_e->pmap;
                if (pvh_e != (pv_hashed_entry_t) pv_h) {
                        pv_hash_remove(pvh_e);
                        pv_h->pmap = pvh_e->pmap;

-                       pv_h->va = pvh_e->va;
+                       pv_h->va_and_flags = pvh_e->va_and_flags;

                        pvh_e->qlink.next = (queue_entry_t) pvh_eh;
                        pvh_eh = pvh_e;
 
                        pvh_e->qlink.next = (queue_entry_t) pvh_eh;
                        pvh_eh = pvh_e;
 


@@ -1636,8 +1866,7 @@ pmap_page_protect_options(
 done:
        UNLOCK_PVH(pai);
 
 done:
        UNLOCK_PVH(pai);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END,
-                  0, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);

 }
 
 
 }
 
 


@@ -1653,11 +1882,11 @@ phys_attribute_clear(
 {
        pv_rooted_entry_t       pv_h;
        pv_hashed_entry_t       pv_e;
 {
        pv_rooted_entry_t       pv_h;
        pv_hashed_entry_t       pv_e;

-       pt_entry_t              *pte;
+       pt_entry_t              *pte = NULL;

        int                     pai;
        pmap_t                  pmap;
        char                    attributes = 0;
        int                     pai;
        pmap_t                  pmap;
        char                    attributes = 0;

-       boolean_t               is_internal, is_reusable, is_ept;
+       boolean_t               is_internal, is_reusable, is_altacct, is_ept;

        int                     ept_bits_to_clear;
        boolean_t               ept_keep_global_mod = FALSE;
 
        int                     ept_bits_to_clear;
        boolean_t               ept_keep_global_mod = FALSE;
 


@@ -1688,8 +1917,7 @@ phys_attribute_clear(
                return;
        }
 
                return;
        }
 

-       PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START,
-                  pn, bits, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);

 
        pv_h = pai_to_pvh(pai);
 
 
        pv_h = pai_to_pvh(pai);
 


@@ -1717,7 +1945,8 @@ phys_attribute_clear(
 
                        pmap = pv_e->pmap;
                        is_ept = is_ept_pmap(pmap);
 
                        pmap = pv_e->pmap;
                        is_ept = is_ept_pmap(pmap);

-                       va = pv_e->va;
+                       is_altacct = IS_ALTACCT_PAGE(pai, pv_e);
+                       va = PVE_VA(pv_e);

                        pte_bits = 0;
 
                        if (bits) {
                        pte_bits = 0;
 
                        if (bits) {


@@ -1781,10 +2010,23 @@ phys_attribute_clear(
                                        /* one more "internal" */
                                        OSAddAtomic(+1, &pmap->stats.internal);
                                        PMAP_STATS_PEAK(pmap->stats.internal);
                                        /* one more "internal" */
                                        OSAddAtomic(+1, &pmap->stats.internal);
                                        PMAP_STATS_PEAK(pmap->stats.internal);

+                                       assert(pmap->stats.internal > 0);
+                                       if (is_altacct) {
+                                               /* no impact on ledgers */
+                                       } else {
+                                               pmap_ledger_credit(pmap,
+                                                                  task_ledgers.internal,
+                                                                  PAGE_SIZE);
+                                               pmap_ledger_credit(
+                                                       pmap,
+                                                       task_ledgers.phys_footprint,
+                                                       PAGE_SIZE);
+                                       }

                                } else {
                                        /* one more "external" */
                                        OSAddAtomic(+1, &pmap->stats.external);
                                        PMAP_STATS_PEAK(pmap->stats.external);
                                } else {
                                        /* one more "external" */
                                        OSAddAtomic(+1, &pmap->stats.external);
                                        PMAP_STATS_PEAK(pmap->stats.external);

+                                       assert(pmap->stats.external > 0);

                                }
                        } else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
                                   !is_reusable &&
                                }
                        } else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
                                   !is_reusable &&


@@ -1792,10 +2034,22 @@ phys_attribute_clear(
                                /* one more "reusable" */
                                OSAddAtomic(+1, &pmap->stats.reusable);
                                PMAP_STATS_PEAK(pmap->stats.reusable);
                                /* one more "reusable" */
                                OSAddAtomic(+1, &pmap->stats.reusable);
                                PMAP_STATS_PEAK(pmap->stats.reusable);

+                               assert(pmap->stats.reusable > 0);

                                if (is_internal) {
                                        /* one less "internal" */
                                        assert(pmap->stats.internal > 0);
                                        OSAddAtomic(-1, &pmap->stats.internal);
                                if (is_internal) {
                                        /* one less "internal" */
                                        assert(pmap->stats.internal > 0);
                                        OSAddAtomic(-1, &pmap->stats.internal);

+                                       if (is_altacct) {
+                                               /* no impact on footprint */
+                                       } else {
+                                               pmap_ledger_debit(pmap,
+                                                                 task_ledgers.internal,
+                                                                 PAGE_SIZE);
+                                               pmap_ledger_debit(
+                                                       pmap,
+                                                       task_ledgers.phys_footprint,
+                                                       PAGE_SIZE);
+                                       }

                                } else {
                                        /* one less "external" */
                                        assert(pmap->stats.external > 0);
                                } else {
                                        /* one less "external" */
                                        assert(pmap->stats.external > 0);


@@ -1833,8 +2087,7 @@ phys_attribute_clear(
 
        UNLOCK_PVH(pai);
 
 
        UNLOCK_PVH(pai);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END,
-                  0, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);

 }
 
 /*
 }
 
 /*


@@ -1900,7 +2153,7 @@ phys_attribute_test(
 
                        pmap = pv_e->pmap;
                        is_ept = is_ept_pmap(pmap);
 
                        pmap = pv_e->pmap;
                        is_ept = is_ept_pmap(pmap);

-                       va = pv_e->va;
+                       va = PVE_VA(pv_e);

                        /*
                         * pick up modify and/or reference bits from mapping
                         */
                        /*
                         * pick up modify and/or reference bits from mapping
                         */


@@ -1942,7 +2195,8 @@ pmap_change_wiring(
        PMAP_LOCK(map);
 
        if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
        PMAP_LOCK(map);
 
        if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)

-               panic("pmap_change_wiring: pte missing");
+               panic("pmap_change_wiring(%p,0x%llx,%d): pte missing",
+                     map, vaddr, wired);

 
        if (wired && !iswired(*pte)) {
                /*
 
        if (wired && !iswired(*pte)) {
                /*


@@ -1980,9 +2234,11 @@ pmap_map_bd(
        unsigned int    flags)
 {
        pt_entry_t      template;
        unsigned int    flags)
 {
        pt_entry_t      template;

-       pt_entry_t      *pte;
-       spl_t           spl;
+       pt_entry_t      *ptep;
+

        vm_offset_t     base = virt;
        vm_offset_t     base = virt;

+       boolean_t       doflush = FALSE;
+

        template = pa_to_pte(start_addr)
                | INTEL_PTE_REF
                | INTEL_PTE_MOD
        template = pa_to_pte(start_addr)
                | INTEL_PTE_REF
                | INTEL_PTE_MOD


@@ -1995,51 +2251,100 @@ pmap_map_bd(
                        template |= INTEL_PTE_PTA;
        }
 
                        template |= INTEL_PTE_PTA;
        }
 

-#if    defined(__x86_64__)

        if ((prot & VM_PROT_EXECUTE) == 0)
                template |= INTEL_PTE_NX;
        if ((prot & VM_PROT_EXECUTE) == 0)
                template |= INTEL_PTE_NX;

-#endif

 
        if (prot & VM_PROT_WRITE)
                template |= INTEL_PTE_WRITE;
 
        while (start_addr < end_addr) {
 
        if (prot & VM_PROT_WRITE)
                template |= INTEL_PTE_WRITE;
 
        while (start_addr < end_addr) {

-               spl = splhigh();
-               pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
-               if (pte == PT_ENTRY_NULL) {
-                       panic("pmap_map_bd: Invalid kernel address\n");
+               ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
+               if (ptep == PT_ENTRY_NULL) {
+                       panic("pmap_map_bd: Invalid kernel address");
+               }
+               if (pte_to_pa(*ptep)) {
+                       doflush = TRUE;

                }
                }

-               pmap_store_pte(pte, template);
-               splx(spl);
+               pmap_store_pte(ptep, template);

                pte_increment_pa(template);
                virt += PAGE_SIZE;
                start_addr += PAGE_SIZE;
        }
                pte_increment_pa(template);
                virt += PAGE_SIZE;
                start_addr += PAGE_SIZE;
        }

-       flush_tlb_raw();
-       PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+       if (doflush) {
+               flush_tlb_raw();
+               PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+       }

        return(virt);
 }
 
        return(virt);
 }
 

-unsigned int
+/* Create a virtual alias beginning at 'ava' of the specified kernel virtual
+ * range. The aliased pagetable range is expanded if
+ * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization,
+ * assumes caller has stabilized the source and destination ranges. Currently
+ * used to populate sections of the trampoline "doublemap" at CPU startup.
+ */
+
+void
+pmap_alias(
+       vm_offset_t     ava,
+       vm_map_offset_t start_addr,
+       vm_map_offset_t end_addr,
+       vm_prot_t       prot,
+       unsigned int    eoptions)
+{
+       pt_entry_t      prot_template, template;
+       pt_entry_t      *aptep, *sptep;
+
+       prot_template =  INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID;
+       if ((prot & VM_PROT_EXECUTE) == 0)
+               prot_template |= INTEL_PTE_NX;
+
+       if (prot & VM_PROT_WRITE)
+               prot_template |= INTEL_PTE_WRITE;
+       assert(((start_addr | end_addr) & PAGE_MASK) == 0);
+       while (start_addr < end_addr) {
+               aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+               if (aptep == PT_ENTRY_NULL) {
+                       if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) {
+                               pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP);
+                               aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+                       } else {
+                               panic("pmap_alias: Invalid alias address");
+                       }
+               }
+               /* The aliased range should not have any active mappings */
+               assert(pte_to_pa(*aptep) == 0);
+
+               sptep = pmap_pte(kernel_pmap, start_addr);
+               assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0));
+               template = pa_to_pte(pte_to_pa(*sptep)) | prot_template;
+               pmap_store_pte(aptep, template);
+
+               ava += PAGE_SIZE;
+               start_addr += PAGE_SIZE;
+       }
+}
+
+mach_vm_size_t

 pmap_query_resident(
        pmap_t          pmap,
        addr64_t        s64,
        addr64_t        e64,
 pmap_query_resident(
        pmap_t          pmap,
        addr64_t        s64,
        addr64_t        e64,

-       unsigned int    *compressed_count_p)
+       mach_vm_size_t  *compressed_bytes_p)

 {
        pt_entry_t     *pde;
        pt_entry_t     *spte, *epte;
        addr64_t        l64;
        uint64_t        deadline;
 {
        pt_entry_t     *pde;
        pt_entry_t     *spte, *epte;
        addr64_t        l64;
        uint64_t        deadline;

-       unsigned int    result;
+       mach_vm_size_t  resident_bytes;
+       mach_vm_size_t  compressed_bytes;

        boolean_t       is_ept;
        boolean_t       is_ept;

-       unsigned int    compressed_count;

 
        pmap_intr_assert();
 
        if (pmap == PMAP_NULL || pmap == kernel_pmap || s64 == e64) {
 
        pmap_intr_assert();
 
        if (pmap == PMAP_NULL || pmap == kernel_pmap || s64 == e64) {

-               if (compressed_count_p) {
-                       *compressed_count_p = 0;
+               if (compressed_bytes_p) {
+                       *compressed_bytes_p = 0;

                }
                return 0;
        }
                }
                return 0;
        }


@@ -2047,12 +2352,11 @@ pmap_query_resident(
        is_ept = is_ept_pmap(pmap);
 
        PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
        is_ept = is_ept_pmap(pmap);
 
        PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,

-                  pmap,
-                  (uint32_t) (s64 >> 32), s64,
-                  (uint32_t) (e64 >> 32), e64);
+                  VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(s64),
+                  VM_KERNEL_ADDRHIDE(e64));

 
 

-       result = 0;
-       compressed_count = 0;
+       resident_bytes = 0;
+       compressed_bytes = 0;

 
        PMAP_LOCK(pmap);
 
 
        PMAP_LOCK(pmap);
 


@@ -2075,9 +2379,9 @@ pmap_query_resident(
 
                                for (; spte < epte; spte++) {
                                        if (pte_to_pa(*spte) != 0) {
 
                                for (; spte < epte; spte++) {
                                        if (pte_to_pa(*spte) != 0) {

-                                               result++;
+                                               resident_bytes += PAGE_SIZE;

                                        } else if (*spte & PTE_COMPRESSED) {
                                        } else if (*spte & PTE_COMPRESSED) {

-                                               compressed_count++;
+                                               compressed_bytes += PAGE_SIZE;

                                        }
                                }
 
                                        }
                                }
 


@@ -2095,20 +2399,96 @@ pmap_query_resident(
        PMAP_UNLOCK(pmap);
 
        PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
        PMAP_UNLOCK(pmap);
 
        PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,

-                  pmap, 0, 0, 0, 0);
+                  resident_bytes);

 
 

-       if (compressed_count_p) {
-               *compressed_count_p = compressed_count;
+       if (compressed_bytes_p) {
+               *compressed_bytes_p = compressed_bytes;

        }
        }

-       return result;
+       return resident_bytes;

 }
 
 }
 

-#if MACH_ASSERT
-void
-pmap_set_process(
-       __unused pmap_t pmap,
-       __unused int pid,
-       __unused char *procname)
+kern_return_t
+pmap_query_page_info(
+       pmap_t          pmap,
+       vm_map_offset_t va,
+       int             *disp_p)
+{
+       int             disp;
+       boolean_t       is_ept;
+       pmap_paddr_t    pa;
+       ppnum_t         pai;
+       pd_entry_t      *pde;
+       pt_entry_t      *pte;
+
+       pmap_intr_assert();
+       if (pmap == PMAP_NULL || pmap == kernel_pmap) {
+               *disp_p = 0;
+               return KERN_INVALID_ARGUMENT;
+       }
+
+       disp = 0;
+       is_ept = is_ept_pmap(pmap);
+
+       PMAP_LOCK(pmap);
+
+       pde = pmap_pde(pmap, va);
+       if (!pde ||
+           !(*pde & PTE_VALID_MASK(is_ept)) ||
+           (*pde & PTE_PS)) {
+               goto done;
+       }
+
+       pte = pmap_pte(pmap, va);
+       if (pte == PT_ENTRY_NULL) {
+               goto done;
+       }
+
+       pa = pte_to_pa(*pte);
+       if (pa == 0) {
+               if (PTE_IS_COMPRESSED(*pte)) {
+                       disp |= PMAP_QUERY_PAGE_COMPRESSED;
+                       if (*pte & PTE_COMPRESSED_ALT) {
+                               disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
+                       }
+               }
+       } else {
+               disp |= PMAP_QUERY_PAGE_PRESENT;
+               pai = pa_index(pa);
+               if (!IS_MANAGED_PAGE(pai)) {
+               } else if (pmap_pv_is_altacct(pmap, va, pai)) {
+                       assert(IS_INTERNAL_PAGE(pai));
+                       disp |= PMAP_QUERY_PAGE_INTERNAL;
+                       disp |= PMAP_QUERY_PAGE_ALTACCT;
+               } else if (IS_REUSABLE_PAGE(pai)) {
+                       disp |= PMAP_QUERY_PAGE_REUSABLE;
+               } else if (IS_INTERNAL_PAGE(pai)) {
+                       disp |= PMAP_QUERY_PAGE_INTERNAL;
+               }
+       }
+
+done:
+       PMAP_UNLOCK(pmap);
+       *disp_p = disp;
+       return KERN_SUCCESS;
+}
+
+void pmap_set_jit_entitled(__unused pmap_t pmap)
+{
+       /* The x86 pmap layer does not care if a map has a JIT entry. */
+       return;
+}
+
+bool pmap_has_prot_policy(__unused vm_prot_t prot)
+{
+       /*
+        * The x86 pmap layer does not apply any policy to any protection
+        * types.
+        */
+       return FALSE;
+}
+
+void pmap_release_pages_fast(void)

 {
 {

+       return;

 }
 }

-#endif /* MACH_ASSERT */
+