#include <vm/pmap.h>
#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
#include <kern/ledger.h>
#include <i386/pmap_internal.h>
panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart);
PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
- (uintptr_t) grand, (uintptr_t) subord,
- (uintptr_t) (va_start>>32), (uintptr_t) va_start, 0);
+ VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
+ VM_KERNEL_ADDRHIDE(va_start));
nvaddr = (vm_map_offset_t)nstart;
num_pde = size >> PDESHIFT;
i += (uint32_t) NPDEPG;
}
else {
- npde = pmap_pde(subord, nstart);
+ npde = pmap_pde(subord, vaddr);
if (npde == 0)
- panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
+ panic("pmap_nest: no npde, subord %p vaddr 0x%llx", subord, vaddr);
tpde = *npde;
- nstart += NBPDE;
pde = pmap_pde(grand, vaddr);
if ((0 == pde) && cpu_64bit) {
PMAP_UNLOCK(grand);
PMAP_UNLOCK(grand);
- PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, KERN_SUCCESS);
return KERN_SUCCESS;
}
*/
kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
-
pd_entry_t *pde;
unsigned int i;
uint64_t num_pde;
uint64_t npdpt = PMAP_INVALID_PDPTNUM;
PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
- (uintptr_t) grand,
- (uintptr_t) (vaddr>>32), (uintptr_t) vaddr, 0, 0);
+ VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));
if ((size & (pmap_nesting_size_min-1)) ||
(vaddr & (pmap_nesting_size_min-1))) {
PMAP_UPDATE_TLBS(grand, va_start, va_end);
PMAP_UNLOCK(grand);
-
- PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, KERN_SUCCESS);
return KERN_SUCCESS;
}
do {
pmap = pv_e->pmap;
- vaddr = pv_e->va;
+ vaddr = PVE_VA(pv_e);
ptep = pmap_pte(pmap, vaddr);
if (0 == ptep)
* insert this page into the given map NOW.
*/
-void
+kern_return_t
pmap_enter(
- register pmap_t pmap,
+ pmap_t pmap,
vm_map_offset_t vaddr,
ppnum_t pn,
vm_prot_t prot,
unsigned int flags,
boolean_t wired)
{
- (void) pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL);
+ return pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL);
}
kern_return_t
pmap_enter_options(
- register pmap_t pmap,
+ pmap_t pmap,
vm_map_offset_t vaddr,
ppnum_t pn,
vm_prot_t prot,
vm_object_t delpage_pm_obj = NULL;
uint64_t delpage_pde_index = 0;
pt_entry_t old_pte;
- kern_return_t kr_expand;
+ kern_return_t kr;
boolean_t is_ept;
+ boolean_t is_altacct;
+
+ kr = KERN_FAILURE;
pmap_intr_assert();
return KERN_INVALID_ARGUMENT;
PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
- pmap,
- (uint32_t) (vaddr >> 32), (uint32_t) vaddr,
- pn, prot);
+ VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(vaddr), pn,
+ prot);
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
set_NX = FALSE;
* pmap is always expanded to include enough hardware
* pages to map one VM page.
*/
- if(superpage) {
+ if (superpage) {
while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) {
/* need room for another pde entry */
PMAP_UNLOCK(pmap);
- kr_expand = pmap_expand_pdpt(pmap, vaddr, options);
- if (kr_expand != KERN_SUCCESS)
- return kr_expand;
+ kr = pmap_expand_pdpt(pmap, vaddr, options);
+ if (kr != KERN_SUCCESS)
+ goto done;
PMAP_LOCK(pmap);
}
} else {
* going to grow pde level page(s)
*/
PMAP_UNLOCK(pmap);
- kr_expand = pmap_expand(pmap, vaddr, options);
- if (kr_expand != KERN_SUCCESS)
- return kr_expand;
+ kr = pmap_expand(pmap, vaddr, options);
+ if (kr != KERN_SUCCESS)
+ goto done;
PMAP_LOCK(pmap);
}
}
if (options & PMAP_EXPAND_OPTIONS_NOENTER) {
PMAP_UNLOCK(pmap);
- return KERN_SUCCESS;
+ kr = KERN_SUCCESS;
+ goto done;
}
if (superpage && *pte && !(*pte & PTE_PS)) {
old_pa_locked = FALSE;
if (old_pa == 0 &&
- (*pte & PTE_COMPRESSED)) {
+ PTE_IS_COMPRESSED(*pte)) {
+ /*
+ * "pmap" should be locked at this point, so this should
+ * not race with another pmap_enter() or pmap_remove_range().
+ */
+ assert(pmap != kernel_pmap);
+
/* one less "compressed" */
OSAddAtomic64(-1, &pmap->stats.compressed);
+ pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
+ PAGE_SIZE);
+ if (*pte & PTE_COMPRESSED_ALT) {
+ pmap_ledger_debit(
+ pmap,
+ task_ledgers.alternate_accounting_compressed,
+ PAGE_SIZE);
+ } else {
+ /* was part of the footprint */
+ pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
+ PAGE_SIZE);
+ }
/* marker will be cleared below */
}
/* Determine delta, PV locked */
need_tlbflush =
((old_attributes ^ template) != PTE_WIRED);
-
+
if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) {
if ((old_attributes ^ template) == PTE_WRITE(is_ept))
need_tlbflush = FALSE;
*/
if (old_pa != (pmap_paddr_t) 0) {
+ boolean_t was_altacct = FALSE;
/*
* Don't do anything to pages outside valid memory here.
/* completely invalidate the PTE */
pmap_store_pte(pte, 0);
+ if (IS_MANAGED_PAGE(pai)) {
+ /*
+ * Remove the mapping from the pvlist for
+ * this physical page.
+ * We'll end up with either a rooted pv or a
+ * hashed pv
+ */
+ pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte, &was_altacct);
+ }
+
if (IS_MANAGED_PAGE(pai)) {
pmap_assert(old_pa_locked == TRUE);
pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
- pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
assert(pmap->stats.resident_count >= 1);
OSAddAtomic(-1, &pmap->stats.resident_count);
if (pmap != kernel_pmap) {
+ /* update pmap stats */
if (IS_REUSABLE_PAGE(pai)) {
- assert(pmap->stats.reusable > 0);
+ PMAP_STATS_ASSERTF(
+ (pmap->stats.reusable > 0,
+ "reusable %d",
+ pmap->stats.reusable));
OSAddAtomic(-1, &pmap->stats.reusable);
} else if (IS_INTERNAL_PAGE(pai)) {
- assert(pmap->stats.internal > 0);
+ PMAP_STATS_ASSERTF(
+ (pmap->stats.internal > 0,
+ "internal %d",
+ pmap->stats.internal));
OSAddAtomic(-1, &pmap->stats.internal);
} else {
- assert(pmap->stats.external > 0);
+ PMAP_STATS_ASSERTF(
+ (pmap->stats.external > 0,
+ "external %d",
+ pmap->stats.external));
OSAddAtomic(-1, &pmap->stats.external);
}
+
+ /* update ledgers */
+ if (was_altacct) {
+ assert(IS_INTERNAL_PAGE(pai));
+ pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+ pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+ } else if (IS_REUSABLE_PAGE(pai)) {
+ assert(!was_altacct);
+ assert(IS_INTERNAL_PAGE(pai));
+ /* was already not in phys_footprint */
+ } else if (IS_INTERNAL_PAGE(pai)) {
+ assert(!was_altacct);
+ assert(!IS_REUSABLE_PAGE(pai));
+ pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+ pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+ } else {
+ /* not an internal page */
+ }
}
if (iswired(*pte)) {
assert(pmap->stats.wired_count >= 1);
pmap_phys_attributes[pai] |= ept_refmod_to_physmap(oattr);
}
- /*
- * Remove the mapping from the pvlist for
- * this physical page.
- * We'll end up with either a rooted pv or a
- * hashed pv
- */
- pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte);
-
} else {
/*
/*
* No mappings yet, use rooted pv
*/
- pv_h->va = vaddr;
+ pv_h->va_and_flags = vaddr;
pv_h->pmap = pmap;
queue_init(&pv_h->qlink);
} else {
pmap_phys_attributes[pai] &= ~PHYS_REUSABLE;
}
+ if ((options & PMAP_OPTIONS_ALT_ACCT) &&
+ IS_INTERNAL_PAGE(pai)) {
+ pv_h->va_and_flags |= PVE_IS_ALTACCT;
+ is_altacct = TRUE;
+ } else {
+ pv_h->va_and_flags &= ~PVE_IS_ALTACCT;
+ is_altacct = FALSE;
+ }
} else {
/*
* Add new pv_hashed_entry after header.
}
}
}
-
+
if (PV_HASHED_ENTRY_NULL == pvh_e)
panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings");
- pvh_e->va = vaddr;
+ pvh_e->va_and_flags = vaddr;
pvh_e->pmap = pmap;
pvh_e->ppn = pn;
+ if ((options & PMAP_OPTIONS_ALT_ACCT) &&
+ IS_INTERNAL_PAGE(pai)) {
+ pvh_e->va_and_flags |= PVE_IS_ALTACCT;
+ is_altacct = TRUE;
+ } else {
+ pvh_e->va_and_flags &= ~PVE_IS_ALTACCT;
+ is_altacct = FALSE;
+ }
pv_hash_add(pvh_e, pv_h);
/*
* for 'managed memory'
*/
pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
- pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
OSAddAtomic(+1, &pmap->stats.resident_count);
if (pmap->stats.resident_count > pmap->stats.resident_max) {
pmap->stats.resident_max = pmap->stats.resident_count;
}
if (pmap != kernel_pmap) {
+ /* update pmap stats */
if (IS_REUSABLE_PAGE(pai)) {
OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
OSAddAtomic(+1, &pmap->stats.external);
PMAP_STATS_PEAK(pmap->stats.external);
}
+
+ /* update ledgers */
+ if (is_altacct) {
+ /* internal but also alternate accounting */
+ assert(IS_INTERNAL_PAGE(pai));
+ pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
+ pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+ /* alternate accounting, so not in footprint */
+ } else if (IS_REUSABLE_PAGE(pai)) {
+ assert(!is_altacct);
+ assert(IS_INTERNAL_PAGE(pai));
+ /* internal but reusable: not in footprint */
+ } else if (IS_INTERNAL_PAGE(pai)) {
+ assert(!is_altacct);
+ assert(!IS_REUSABLE_PAGE(pai));
+ /* internal: add to footprint */
+ pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
+ pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+ } else {
+ /* not internal: not in footprint */
+ }
}
} else if (last_managed_page == 0) {
/* Account for early mappings created before "managed pages"
* are determined. Consider consulting the available DRAM map.
*/
pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
- pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
OSAddAtomic(+1, &pmap->stats.resident_count);
if (pmap != kernel_pmap) {
#if 00
PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
}
- PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, 0, 0, 0, 0, 0);
- return KERN_SUCCESS;
+ kr = KERN_SUCCESS;
+done:
+ PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
+ return kr;
}
/*
pv_hashed_entry_t pvh_e;
int pvh_cnt = 0;
int num_removed, num_unwired, num_found, num_invalid;
- int num_device, num_external, num_internal, num_reusable;
- uint64_t num_compressed;
+ int stats_external, stats_internal, stats_reusable;
+ uint64_t stats_compressed;
+ int ledgers_internal, ledgers_alt_internal;
+ uint64_t ledgers_compressed, ledgers_alt_compressed;
ppnum_t pai;
pmap_paddr_t pa;
vm_map_offset_t vaddr;
boolean_t is_ept = is_ept_pmap(pmap);
+ boolean_t was_altacct;
num_removed = 0;
num_unwired = 0;
num_found = 0;
num_invalid = 0;
- num_device = 0;
- num_external = 0;
- num_internal = 0;
- num_reusable = 0;
- num_compressed = 0;
+ stats_external = 0;
+ stats_internal = 0;
+ stats_reusable = 0;
+ stats_compressed = 0;
+ ledgers_internal = 0;
+ ledgers_compressed = 0;
+ ledgers_alt_internal = 0;
+ ledgers_alt_compressed = 0;
/* invalidate the PTEs first to "freeze" them */
for (cpte = spte, vaddr = start_vaddr;
cpte < epte;
pa = pte_to_pa(p);
if (pa == 0) {
- if (pmap != kernel_pmap &&
- (options & PMAP_OPTIONS_REMOVE) &&
- (p & PTE_COMPRESSED)) {
- /* one less "compressed" */
- num_compressed++;
- /* clear marker */
+ if ((options & PMAP_OPTIONS_REMOVE) &&
+ (PTE_IS_COMPRESSED(p))) {
+ assert(pmap != kernel_pmap);
+ /* one less "compressed"... */
+ stats_compressed++;
+ ledgers_compressed++;
+ if (p & PTE_COMPRESSED_ALT) {
+ /* ... but it used to be "ALTACCT" */
+ ledgers_alt_compressed++;
+ }
+ /* clear marker(s) */
/* XXX probably does not need to be atomic! */
- pmap_update_pte(cpte, PTE_COMPRESSED, 0);
+ pmap_update_pte(cpte, INTEL_PTE_COMPRESSED_MASK, 0);
}
continue;
}
* Just remove the mappings.
*/
pmap_store_pte(cpte, 0);
- num_device++;
continue;
}
cpte++, vaddr += PAGE_SIZE_64) {
pa = pte_to_pa(*cpte);
- if (pa == 0)
+ if (pa == 0) {
+ check_pte_for_compressed_marker:
+ /*
+ * This PTE could have been replaced with a
+ * "compressed" marker after our first "freeze"
+ * loop above, so check again.
+ */
+ if ((options & PMAP_OPTIONS_REMOVE) &&
+ (PTE_IS_COMPRESSED(*cpte))) {
+ assert(pmap != kernel_pmap);
+ /* one less "compressed"... */
+ stats_compressed++;
+ ledgers_compressed++;
+ if (*cpte & PTE_COMPRESSED_ALT) {
+ /* ... but it used to be "ALTACCT" */
+ ledgers_alt_compressed++;
+ }
+ pmap_store_pte(cpte, 0);
+ }
continue;
+ }
pai = pa_index(pa);
pa = pte_to_pa(*cpte);
if (pa == 0) {
UNLOCK_PVH(pai);
- continue;
+ goto check_pte_for_compressed_marker;
}
+
+ /*
+ * Remove the mapping from the pvlist for this physical page.
+ */
+ pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte, &was_altacct);
+
num_removed++;
+ /* update pmap stats */
if (IS_REUSABLE_PAGE(pai)) {
- num_reusable++;
+ stats_reusable++;
+ } else if (IS_INTERNAL_PAGE(pai)) {
+ stats_internal++;
+ } else {
+ stats_external++;
+ }
+ /* update ledgers */
+ if (was_altacct) {
+ /* internal and alternate accounting */
+ assert(IS_INTERNAL_PAGE(pai));
+ ledgers_internal++;
+ ledgers_alt_internal++;
+ } else if (IS_REUSABLE_PAGE(pai)) {
+ /* internal but reusable */
+ assert(!was_altacct);
+ assert(IS_INTERNAL_PAGE(pai));
} else if (IS_INTERNAL_PAGE(pai)) {
- num_internal++;
+ /* internal */
+ assert(!was_altacct);
+ assert(!IS_REUSABLE_PAGE(pai));
+ ledgers_internal++;
} else {
- num_external++;
+ /* not internal */
}
/*
* nuke the entry in the page table
*/
/* remember reference and change */
- pmap_phys_attributes[pai] |=
- (char) (*cpte & (PHYS_MODIFIED | PHYS_REFERENCED));
-
- /*
- * Remove the mapping from the pvlist for this physical page.
- */
- pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte);
+ if (!is_ept) {
+ pmap_phys_attributes[pai] |=
+ *cpte & (PHYS_MODIFIED | PHYS_REFERENCED);
+ } else {
+ pmap_phys_attributes[pai] |=
+ ept_refmod_to_physmap((*cpte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+ }
/* completely invalidate the PTE */
pmap_store_pte(cpte, 0);
panic("pmap_remove_range: resident_count");
#endif
pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));
- pmap_ledger_debit(pmap, task_ledgers.phys_footprint, machine_ptob(num_removed));
- assert(pmap->stats.resident_count >= num_removed);
+ PMAP_STATS_ASSERTF((pmap->stats.resident_count >= num_removed,
+ "pmap=%p num_removed=%d stats.resident_count=%d",
+ pmap, num_removed, pmap->stats.resident_count));
OSAddAtomic(-num_removed, &pmap->stats.resident_count);
if (pmap != kernel_pmap) {
-#if 00
- assert(pmap->stats.device >= num_device);
- if (num_device)
- OSAddAtomic(-num_device, &pmap->stats.device);
-#endif /* 00 */
- assert(pmap->stats.external >= num_external);
- if (num_external)
- OSAddAtomic(-num_external, &pmap->stats.external);
- assert(pmap->stats.internal >= num_internal);
- if (num_internal)
- OSAddAtomic(-num_internal, &pmap->stats.internal);
- assert(pmap->stats.reusable >= num_reusable);
- if (num_reusable)
- OSAddAtomic(-num_reusable, &pmap->stats.reusable);
- assert(pmap->stats.compressed >= num_compressed);
- if (num_compressed)
- OSAddAtomic64(-num_compressed, &pmap->stats.compressed);
+ PMAP_STATS_ASSERTF((pmap->stats.external >= stats_external,
+ "pmap=%p stats_external=%d stats.external=%d",
+ pmap, stats_external, pmap->stats.external));
+ PMAP_STATS_ASSERTF((pmap->stats.internal >= stats_internal,
+ "pmap=%p stats_internal=%d stats.internal=%d",
+ pmap, stats_internal, pmap->stats.internal));
+ PMAP_STATS_ASSERTF((pmap->stats.reusable >= stats_reusable,
+ "pmap=%p stats_reusable=%d stats.reusable=%d",
+ pmap, stats_reusable, pmap->stats.reusable));
+ PMAP_STATS_ASSERTF((pmap->stats.compressed >= stats_compressed,
+ "pmap=%p stats_compressed=%lld, stats.compressed=%lld",
+ pmap, stats_compressed, pmap->stats.compressed));
+
+ /* update pmap stats */
+ if (stats_external) {
+ OSAddAtomic(-stats_external, &pmap->stats.external);
+ }
+ if (stats_internal) {
+ OSAddAtomic(-stats_internal, &pmap->stats.internal);
+ }
+ if (stats_reusable)
+ OSAddAtomic(-stats_reusable, &pmap->stats.reusable);
+ if (stats_compressed)
+ OSAddAtomic64(-stats_compressed, &pmap->stats.compressed);
+ /* update ledgers */
+ if (ledgers_internal) {
+ pmap_ledger_debit(pmap,
+ task_ledgers.internal,
+ machine_ptob(ledgers_internal));
+ }
+ if (ledgers_compressed) {
+ pmap_ledger_debit(pmap,
+ task_ledgers.internal_compressed,
+ machine_ptob(ledgers_compressed));
+ }
+ if (ledgers_alt_internal) {
+ pmap_ledger_debit(pmap,
+ task_ledgers.alternate_accounting,
+ machine_ptob(ledgers_alt_internal));
+ }
+ if (ledgers_alt_compressed) {
+ pmap_ledger_debit(pmap,
+ task_ledgers.alternate_accounting_compressed,
+ machine_ptob(ledgers_alt_compressed));
+ }
+ pmap_ledger_debit(pmap,
+ task_ledgers.phys_footprint,
+ machine_ptob((ledgers_internal -
+ ledgers_alt_internal) +
+ (ledgers_compressed -
+ ledgers_alt_compressed)));
}
#if TESTING
if (pmap->stats.wired_count < num_unwired)
panic("pmap_remove_range: wired_count");
#endif
- assert(pmap->stats.wired_count >= num_unwired);
+ PMAP_STATS_ASSERTF((pmap->stats.wired_count >= num_unwired,
+ "pmap=%p num_unwired=%d stats.wired_count=%d",
+ pmap, num_unwired, pmap->stats.wired_count));
OSAddAtomic(-num_unwired, &pmap->stats.wired_count);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
is_ept = is_ept_pmap(map);
PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
- map,
- (uint32_t) (s64 >> 32), s64,
- (uint32_t) (e64 >> 32), e64);
-
+ VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(s64),
+ VM_KERNEL_ADDRHIDE(e64));
PMAP_LOCK(map);
PMAP_UNLOCK(map);
- PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END,
- map, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);
}
*/
return;
}
- PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START,
- pn, prot, 0, 0, 0);
+
+ PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, pn, prot);
/*
* Determine the new protection.
pmap = pv_e->pmap;
is_ept = is_ept_pmap(pmap);
- vaddr = pv_e->va;
+ vaddr = PVE_VA(pv_e);
pte = pmap_pte(pmap, vaddr);
pmap_assert2((pa_index(pte_to_pa(*pte)) == pn),
if (pmap != kernel_pmap &&
(options & PMAP_OPTIONS_COMPRESSOR) &&
IS_INTERNAL_PAGE(pai)) {
- /* mark this PTE as having been "reclaimed" */
+ assert(!PTE_IS_COMPRESSED(*pte));
+ /* mark this PTE as having been "compressed" */
new_pte_value = PTE_COMPRESSED;
+ if (IS_ALTACCT_PAGE(pai, pv_e)) {
+ new_pte_value |= PTE_COMPRESSED_ALT;
+ }
} else {
new_pte_value = 0;
}
pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0);
PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE);
+ if (!is_ept) {
+ pmap_phys_attributes[pai] |=
+ *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
+ } else {
+ pmap_phys_attributes[pai] |=
+ ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+ }
if ((options &
PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) &&
- ! (pmap_phys_attributes[pai] &
- PHYS_MODIFIED) &&
- (*pte & PHYS_MODIFIED)) {
+ IS_INTERNAL_PAGE(pai) &&
+ (pmap_phys_attributes[pai] &
+ PHYS_MODIFIED)) {
/*
* Page is actually "modified" and
* will be compressed. Start
* accounting for it as "compressed".
*/
+ assert(!(options & PMAP_OPTIONS_COMPRESSOR));
options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
options |= PMAP_OPTIONS_COMPRESSOR;
- new_pte_value = PTE_COMPRESSED;
- }
- if (!is_ept) {
- pmap_phys_attributes[pai] |=
- *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
- } else {
- pmap_phys_attributes[pai] |=
- ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED);
+ assert(new_pte_value == 0);
+ if (pmap != kernel_pmap) {
+ new_pte_value = PTE_COMPRESSED;
+ if (IS_ALTACCT_PAGE(pai, pv_e)) {
+ new_pte_value |= PTE_COMPRESSED_ALT;
+ }
+ }
}
pmap_store_pte(pte, new_pte_value);
}
- if (new_pte_value == PTE_COMPRESSED) {
- /* one more "compressed" page */
- OSAddAtomic64(+1, &pmap->stats.compressed);
- PMAP_STATS_PEAK(pmap->stats.compressed);
- pmap->stats.compressed_lifetime++;
- }
-
#if TESTING
if (pmap->stats.resident_count < 1)
panic("pmap_page_protect: resident_count");
pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
assert(pmap->stats.resident_count >= 1);
OSAddAtomic(-1, &pmap->stats.resident_count);
+
+ /*
+ * We only ever compress internal pages.
+ */
if (options & PMAP_OPTIONS_COMPRESSOR) {
- /*
- * This removal is only being done so we can send this page to
- * the compressor; therefore it mustn't affect total task footprint.
- */
- pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
- } else {
- pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+ assert(IS_INTERNAL_PAGE(pai));
}
-
if (pmap != kernel_pmap) {
+ /* update pmap stats */
if (IS_REUSABLE_PAGE(pai)) {
assert(pmap->stats.reusable > 0);
OSAddAtomic(-1, &pmap->stats.reusable);
assert(pmap->stats.external > 0);
OSAddAtomic(-1, &pmap->stats.external);
}
+ if ((options & PMAP_OPTIONS_COMPRESSOR) &&
+ IS_INTERNAL_PAGE(pai)) {
+ /* adjust "compressed" stats */
+ OSAddAtomic64(+1, &pmap->stats.compressed);
+ PMAP_STATS_PEAK(pmap->stats.compressed);
+ pmap->stats.compressed_lifetime++;
+ }
+
+ /* update ledgers */
+ if (IS_ALTACCT_PAGE(pai, pv_e)) {
+ assert(IS_INTERNAL_PAGE(pai));
+ pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+ pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
+ if (options & PMAP_OPTIONS_COMPRESSOR) {
+ pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+ pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE);
+ }
+ } else if (IS_REUSABLE_PAGE(pai)) {
+ assert(!IS_ALTACCT_PAGE(pai, pv_e));
+ assert(IS_INTERNAL_PAGE(pai));
+ if (options & PMAP_OPTIONS_COMPRESSOR) {
+ pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+ /* was not in footprint, but is now */
+ pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+ }
+ } else if (IS_INTERNAL_PAGE(pai)) {
+ assert(!IS_ALTACCT_PAGE(pai, pv_e));
+ assert(!IS_REUSABLE_PAGE(pai));
+ pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
+ /*
+ * Update all stats related to physical
+ * footprint, which only deals with
+ * internal pages.
+ */
+ if (options & PMAP_OPTIONS_COMPRESSOR) {
+ /*
+ * This removal is only being
+ * done so we can send this page
+ * to the compressor; therefore
+ * it mustn't affect total task
+ * footprint.
+ */
+ pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
+ } else {
+ /*
+ * This internal page isn't
+ * going to the compressor,
+ * so adjust stats to keep
+ * phys_footprint up to date.
+ */
+ pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
+ }
+ }
}
/*
if (pvh_e != (pv_hashed_entry_t) pv_h) {
pv_hash_remove(pvh_e);
pv_h->pmap = pvh_e->pmap;
- pv_h->va = pvh_e->va;
+ pv_h->va_and_flags = pvh_e->va_and_flags;
pvh_e->qlink.next = (queue_entry_t) pvh_eh;
pvh_eh = pvh_e;
done:
UNLOCK_PVH(pai);
- PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);
}
{
pv_rooted_entry_t pv_h;
pv_hashed_entry_t pv_e;
- pt_entry_t *pte;
+ pt_entry_t *pte = NULL;
int pai;
pmap_t pmap;
char attributes = 0;
- boolean_t is_internal, is_reusable, is_ept;
+ boolean_t is_internal, is_reusable, is_altacct, is_ept;
int ept_bits_to_clear;
boolean_t ept_keep_global_mod = FALSE;
return;
}
- PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START,
- pn, bits, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);
pv_h = pai_to_pvh(pai);
pmap = pv_e->pmap;
is_ept = is_ept_pmap(pmap);
- va = pv_e->va;
+ is_altacct = IS_ALTACCT_PAGE(pai, pv_e);
+ va = PVE_VA(pv_e);
pte_bits = 0;
if (bits) {
/* one more "internal" */
OSAddAtomic(+1, &pmap->stats.internal);
PMAP_STATS_PEAK(pmap->stats.internal);
+ assert(pmap->stats.internal > 0);
+ if (is_altacct) {
+ /* no impact on ledgers */
+ } else {
+ pmap_ledger_credit(pmap,
+ task_ledgers.internal,
+ PAGE_SIZE);
+ pmap_ledger_credit(
+ pmap,
+ task_ledgers.phys_footprint,
+ PAGE_SIZE);
+ }
} else {
/* one more "external" */
OSAddAtomic(+1, &pmap->stats.external);
PMAP_STATS_PEAK(pmap->stats.external);
+ assert(pmap->stats.external > 0);
}
} else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
!is_reusable &&
/* one more "reusable" */
OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
+ assert(pmap->stats.reusable > 0);
if (is_internal) {
/* one less "internal" */
assert(pmap->stats.internal > 0);
OSAddAtomic(-1, &pmap->stats.internal);
+ if (is_altacct) {
+ /* no impact on footprint */
+ } else {
+ pmap_ledger_debit(pmap,
+ task_ledgers.internal,
+ PAGE_SIZE);
+ pmap_ledger_debit(
+ pmap,
+ task_ledgers.phys_footprint,
+ PAGE_SIZE);
+ }
} else {
/* one less "external" */
assert(pmap->stats.external > 0);
UNLOCK_PVH(pai);
- PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);
}
/*
pmap = pv_e->pmap;
is_ept = is_ept_pmap(pmap);
- va = pv_e->va;
+ va = PVE_VA(pv_e);
/*
* pick up modify and/or reference bits from mapping
*/
PMAP_LOCK(map);
if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
- panic("pmap_change_wiring: pte missing");
+ panic("pmap_change_wiring(%p,0x%llx,%d): pte missing",
+ map, vaddr, wired);
if (wired && !iswired(*pte)) {
/*
unsigned int flags)
{
pt_entry_t template;
- pt_entry_t *pte;
- spl_t spl;
+ pt_entry_t *ptep;
+
vm_offset_t base = virt;
+ boolean_t doflush = FALSE;
+
template = pa_to_pte(start_addr)
| INTEL_PTE_REF
| INTEL_PTE_MOD
template |= INTEL_PTE_PTA;
}
-#if defined(__x86_64__)
if ((prot & VM_PROT_EXECUTE) == 0)
template |= INTEL_PTE_NX;
-#endif
if (prot & VM_PROT_WRITE)
template |= INTEL_PTE_WRITE;
while (start_addr < end_addr) {
- spl = splhigh();
- pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
- if (pte == PT_ENTRY_NULL) {
- panic("pmap_map_bd: Invalid kernel address\n");
+ ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
+ if (ptep == PT_ENTRY_NULL) {
+ panic("pmap_map_bd: Invalid kernel address");
+ }
+ if (pte_to_pa(*ptep)) {
+ doflush = TRUE;
}
- pmap_store_pte(pte, template);
- splx(spl);
+ pmap_store_pte(ptep, template);
pte_increment_pa(template);
virt += PAGE_SIZE;
start_addr += PAGE_SIZE;
}
- flush_tlb_raw();
- PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+ if (doflush) {
+ flush_tlb_raw();
+ PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+ }
return(virt);
}
-unsigned int
+/* Create a virtual alias beginning at 'ava' of the specified kernel virtual
+ * range. The aliased pagetable range is expanded if
+ * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization,
+ * assumes caller has stabilized the source and destination ranges. Currently
+ * used to populate sections of the trampoline "doublemap" at CPU startup.
+ */
+
+void
+pmap_alias(
+ vm_offset_t ava,
+ vm_map_offset_t start_addr,
+ vm_map_offset_t end_addr,
+ vm_prot_t prot,
+ unsigned int eoptions)
+{
+ pt_entry_t prot_template, template;
+ pt_entry_t *aptep, *sptep;
+
+ prot_template = INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID;
+ if ((prot & VM_PROT_EXECUTE) == 0)
+ prot_template |= INTEL_PTE_NX;
+
+ if (prot & VM_PROT_WRITE)
+ prot_template |= INTEL_PTE_WRITE;
+ assert(((start_addr | end_addr) & PAGE_MASK) == 0);
+ while (start_addr < end_addr) {
+ aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+ if (aptep == PT_ENTRY_NULL) {
+ if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) {
+ pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP);
+ aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+ } else {
+ panic("pmap_alias: Invalid alias address");
+ }
+ }
+ /* The aliased range should not have any active mappings */
+ assert(pte_to_pa(*aptep) == 0);
+
+ sptep = pmap_pte(kernel_pmap, start_addr);
+ assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0));
+ template = pa_to_pte(pte_to_pa(*sptep)) | prot_template;
+ pmap_store_pte(aptep, template);
+
+ ava += PAGE_SIZE;
+ start_addr += PAGE_SIZE;
+ }
+}
+
+mach_vm_size_t
pmap_query_resident(
pmap_t pmap,
addr64_t s64,
addr64_t e64,
- unsigned int *compressed_count_p)
+ mach_vm_size_t *compressed_bytes_p)
{
pt_entry_t *pde;
pt_entry_t *spte, *epte;
addr64_t l64;
uint64_t deadline;
- unsigned int result;
+ mach_vm_size_t resident_bytes;
+ mach_vm_size_t compressed_bytes;
boolean_t is_ept;
- unsigned int compressed_count;
pmap_intr_assert();
if (pmap == PMAP_NULL || pmap == kernel_pmap || s64 == e64) {
- if (compressed_count_p) {
- *compressed_count_p = 0;
+ if (compressed_bytes_p) {
+ *compressed_bytes_p = 0;
}
return 0;
}
is_ept = is_ept_pmap(pmap);
PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
- pmap,
- (uint32_t) (s64 >> 32), s64,
- (uint32_t) (e64 >> 32), e64);
+ VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(s64),
+ VM_KERNEL_ADDRHIDE(e64));
- result = 0;
- compressed_count = 0;
+ resident_bytes = 0;
+ compressed_bytes = 0;
PMAP_LOCK(pmap);
for (; spte < epte; spte++) {
if (pte_to_pa(*spte) != 0) {
- result++;
+ resident_bytes += PAGE_SIZE;
} else if (*spte & PTE_COMPRESSED) {
- compressed_count++;
+ compressed_bytes += PAGE_SIZE;
}
}
PMAP_UNLOCK(pmap);
PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
- pmap, 0, 0, 0, 0);
+ resident_bytes);
- if (compressed_count_p) {
- *compressed_count_p = compressed_count;
+ if (compressed_bytes_p) {
+ *compressed_bytes_p = compressed_bytes;
}
- return result;
+ return resident_bytes;
}
-#if MACH_ASSERT
-void
-pmap_set_process(
- __unused pmap_t pmap,
- __unused int pid,
- __unused char *procname)
+kern_return_t
+pmap_query_page_info(
+ pmap_t pmap,
+ vm_map_offset_t va,
+ int *disp_p)
+{
+ int disp;
+ boolean_t is_ept;
+ pmap_paddr_t pa;
+ ppnum_t pai;
+ pd_entry_t *pde;
+ pt_entry_t *pte;
+
+ pmap_intr_assert();
+ if (pmap == PMAP_NULL || pmap == kernel_pmap) {
+ *disp_p = 0;
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ disp = 0;
+ is_ept = is_ept_pmap(pmap);
+
+ PMAP_LOCK(pmap);
+
+ pde = pmap_pde(pmap, va);
+ if (!pde ||
+ !(*pde & PTE_VALID_MASK(is_ept)) ||
+ (*pde & PTE_PS)) {
+ goto done;
+ }
+
+ pte = pmap_pte(pmap, va);
+ if (pte == PT_ENTRY_NULL) {
+ goto done;
+ }
+
+ pa = pte_to_pa(*pte);
+ if (pa == 0) {
+ if (PTE_IS_COMPRESSED(*pte)) {
+ disp |= PMAP_QUERY_PAGE_COMPRESSED;
+ if (*pte & PTE_COMPRESSED_ALT) {
+ disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
+ }
+ }
+ } else {
+ disp |= PMAP_QUERY_PAGE_PRESENT;
+ pai = pa_index(pa);
+ if (!IS_MANAGED_PAGE(pai)) {
+ } else if (pmap_pv_is_altacct(pmap, va, pai)) {
+ assert(IS_INTERNAL_PAGE(pai));
+ disp |= PMAP_QUERY_PAGE_INTERNAL;
+ disp |= PMAP_QUERY_PAGE_ALTACCT;
+ } else if (IS_REUSABLE_PAGE(pai)) {
+ disp |= PMAP_QUERY_PAGE_REUSABLE;
+ } else if (IS_INTERNAL_PAGE(pai)) {
+ disp |= PMAP_QUERY_PAGE_INTERNAL;
+ }
+ }
+
+done:
+ PMAP_UNLOCK(pmap);
+ *disp_p = disp;
+ return KERN_SUCCESS;
+}
+
+void pmap_set_jit_entitled(__unused pmap_t pmap)
+{
+ /* The x86 pmap layer does not care if a map has a JIT entry. */
+ return;
+}
+
+bool pmap_has_prot_policy(__unused vm_prot_t prot)
+{
+ /*
+ * The x86 pmap layer does not apply any policy to any protection
+ * types.
+ */
+ return FALSE;
+}
+
+void pmap_release_pages_fast(void)
{
+ return;
}
-#endif /* MACH_ASSERT */
+
projects / apple / xnu.git / blobdiff