X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..5c9f46613a83ebfc29a5b1f099448259e96a98f0:/osfmk/i386/pmap_x86_common.c?ds=sidebyside diff --git a/osfmk/i386/pmap_x86_common.c b/osfmk/i386/pmap_x86_common.c index 9841a0754..b66630233 100644 --- a/osfmk/i386/pmap_x86_common.c +++ b/osfmk/i386/pmap_x86_common.c @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -111,8 +112,8 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t panic("pmap_nest: va_start(0x%llx) != nstart(0x%llx)\n", va_start, nstart); PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START, - (uintptr_t) grand, (uintptr_t) subord, - (uintptr_t) (va_start>>32), (uintptr_t) va_start, 0); + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord), + VM_KERNEL_ADDRHIDE(va_start)); nvaddr = (vm_map_offset_t)nstart; num_pde = size >> PDESHIFT; @@ -178,11 +179,10 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t i += (uint32_t) NPDEPG; } else { - npde = pmap_pde(subord, nstart); + npde = pmap_pde(subord, vaddr); if (npde == 0) - panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart); + panic("pmap_nest: no npde, subord %p vaddr 0x%llx", subord, vaddr); tpde = *npde; - nstart += NBPDE; pde = pmap_pde(grand, vaddr); if ((0 == pde) && cpu_64bit) { PMAP_UNLOCK(grand); @@ -201,7 +201,7 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t PMAP_UNLOCK(grand); - PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, KERN_SUCCESS); return KERN_SUCCESS; } @@ -216,7 +216,6 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t va_start, addr64_t */ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { - pd_entry_t *pde; unsigned int i; uint64_t num_pde; @@ -224,8 +223,7 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { uint64_t npdpt = PMAP_INVALID_PDPTNUM; PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START, - (uintptr_t) grand, - (uintptr_t) (vaddr>>32), (uintptr_t) vaddr, 0, 0); + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr)); if ((size & (pmap_nesting_size_min-1)) || (vaddr & (pmap_nesting_size_min-1))) { @@ -267,8 +265,8 @@ kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { PMAP_UPDATE_TLBS(grand, va_start, va_end); PMAP_UNLOCK(grand); - - PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, KERN_SUCCESS); return KERN_SUCCESS; } @@ -405,7 +403,7 @@ pmap_update_cache_attributes_locked(ppnum_t pn, unsigned attributes) { do { pmap = pv_e->pmap; - vaddr = pv_e->va; + vaddr = PVE_VA(pv_e); ptep = pmap_pte(pmap, vaddr); if (0 == ptep) @@ -452,9 +450,9 @@ void x86_filter_TLB_coherency_interrupts(boolean_t dofilter) { * insert this page into the given map NOW. */ -void +kern_return_t pmap_enter( - register pmap_t pmap, + pmap_t pmap, vm_map_offset_t vaddr, ppnum_t pn, vm_prot_t prot, @@ -462,13 +460,13 @@ pmap_enter( unsigned int flags, boolean_t wired) { - (void) pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL); + return pmap_enter_options(pmap, vaddr, pn, prot, fault_type, flags, wired, PMAP_EXPAND_OPTIONS_NONE, NULL); } kern_return_t pmap_enter_options( - register pmap_t pmap, + pmap_t pmap, vm_map_offset_t vaddr, ppnum_t pn, vm_prot_t prot, @@ -495,8 +493,11 @@ pmap_enter_options( vm_object_t delpage_pm_obj = NULL; uint64_t delpage_pde_index = 0; pt_entry_t old_pte; - kern_return_t kr_expand; + kern_return_t kr; boolean_t is_ept; + boolean_t is_altacct; + + kr = KERN_FAILURE; pmap_intr_assert(); @@ -514,9 +515,8 @@ pmap_enter_options( return KERN_INVALID_ARGUMENT; PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START, - pmap, - (uint32_t) (vaddr >> 32), (uint32_t) vaddr, - pn, prot); + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(vaddr), pn, + prot); if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) set_NX = FALSE; @@ -546,13 +546,13 @@ Retry: * pmap is always expanded to include enough hardware * pages to map one VM page. */ - if(superpage) { + if (superpage) { while ((pte = pmap64_pde(pmap, vaddr)) == PD_ENTRY_NULL) { /* need room for another pde entry */ PMAP_UNLOCK(pmap); - kr_expand = pmap_expand_pdpt(pmap, vaddr, options); - if (kr_expand != KERN_SUCCESS) - return kr_expand; + kr = pmap_expand_pdpt(pmap, vaddr, options); + if (kr != KERN_SUCCESS) + goto done; PMAP_LOCK(pmap); } } else { @@ -562,15 +562,16 @@ Retry: * going to grow pde level page(s) */ PMAP_UNLOCK(pmap); - kr_expand = pmap_expand(pmap, vaddr, options); - if (kr_expand != KERN_SUCCESS) - return kr_expand; + kr = pmap_expand(pmap, vaddr, options); + if (kr != KERN_SUCCESS) + goto done; PMAP_LOCK(pmap); } } if (options & PMAP_EXPAND_OPTIONS_NOENTER) { PMAP_UNLOCK(pmap); - return KERN_SUCCESS; + kr = KERN_SUCCESS; + goto done; } if (superpage && *pte && !(*pte & PTE_PS)) { @@ -590,9 +591,27 @@ Retry: old_pa_locked = FALSE; if (old_pa == 0 && - (*pte & PTE_COMPRESSED)) { + PTE_IS_COMPRESSED(*pte)) { + /* + * "pmap" should be locked at this point, so this should + * not race with another pmap_enter() or pmap_remove_range(). + */ + assert(pmap != kernel_pmap); + /* one less "compressed" */ OSAddAtomic64(-1, &pmap->stats.compressed); + pmap_ledger_debit(pmap, task_ledgers.internal_compressed, + PAGE_SIZE); + if (*pte & PTE_COMPRESSED_ALT) { + pmap_ledger_debit( + pmap, + task_ledgers.alternate_accounting_compressed, + PAGE_SIZE); + } else { + /* was part of the footprint */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, + PAGE_SIZE); + } /* marker will be cleared below */ } @@ -690,7 +709,7 @@ Retry: /* Determine delta, PV locked */ need_tlbflush = ((old_attributes ^ template) != PTE_WIRED); - + if (need_tlbflush == TRUE && !(old_attributes & PTE_WRITE(is_ept))) { if ((old_attributes ^ template) == PTE_WRITE(is_ept)) need_tlbflush = FALSE; @@ -733,6 +752,7 @@ dont_update_pte: */ if (old_pa != (pmap_paddr_t) 0) { + boolean_t was_altacct = FALSE; /* * Don't do anything to pages outside valid memory here. @@ -750,23 +770,60 @@ dont_update_pte: /* completely invalidate the PTE */ pmap_store_pte(pte, 0); + if (IS_MANAGED_PAGE(pai)) { + /* + * Remove the mapping from the pvlist for + * this physical page. + * We'll end up with either a rooted pv or a + * hashed pv + */ + pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte, &was_altacct); + } + if (IS_MANAGED_PAGE(pai)) { pmap_assert(old_pa_locked == TRUE); pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE); - pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); assert(pmap->stats.resident_count >= 1); OSAddAtomic(-1, &pmap->stats.resident_count); if (pmap != kernel_pmap) { + /* update pmap stats */ if (IS_REUSABLE_PAGE(pai)) { - assert(pmap->stats.reusable > 0); + PMAP_STATS_ASSERTF( + (pmap->stats.reusable > 0, + "reusable %d", + pmap->stats.reusable)); OSAddAtomic(-1, &pmap->stats.reusable); } else if (IS_INTERNAL_PAGE(pai)) { - assert(pmap->stats.internal > 0); + PMAP_STATS_ASSERTF( + (pmap->stats.internal > 0, + "internal %d", + pmap->stats.internal)); OSAddAtomic(-1, &pmap->stats.internal); } else { - assert(pmap->stats.external > 0); + PMAP_STATS_ASSERTF( + (pmap->stats.external > 0, + "external %d", + pmap->stats.external)); OSAddAtomic(-1, &pmap->stats.external); } + + /* update ledgers */ + if (was_altacct) { + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + } else if (IS_REUSABLE_PAGE(pai)) { + assert(!was_altacct); + assert(IS_INTERNAL_PAGE(pai)); + /* was already not in phys_footprint */ + } else if (IS_INTERNAL_PAGE(pai)) { + assert(!was_altacct); + assert(!IS_REUSABLE_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } else { + /* not an internal page */ + } } if (iswired(*pte)) { assert(pmap->stats.wired_count >= 1); @@ -781,14 +838,6 @@ dont_update_pte: pmap_phys_attributes[pai] |= ept_refmod_to_physmap(oattr); } - /* - * Remove the mapping from the pvlist for - * this physical page. - * We'll end up with either a rooted pv or a - * hashed pv - */ - pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, &old_pte); - } else { /* @@ -833,7 +882,7 @@ dont_update_pte: /* * No mappings yet, use rooted pv */ - pv_h->va = vaddr; + pv_h->va_and_flags = vaddr; pv_h->pmap = pmap; queue_init(&pv_h->qlink); @@ -847,6 +896,14 @@ dont_update_pte: } else { pmap_phys_attributes[pai] &= ~PHYS_REUSABLE; } + if ((options & PMAP_OPTIONS_ALT_ACCT) && + IS_INTERNAL_PAGE(pai)) { + pv_h->va_and_flags |= PVE_IS_ALTACCT; + is_altacct = TRUE; + } else { + pv_h->va_and_flags &= ~PVE_IS_ALTACCT; + is_altacct = FALSE; + } } else { /* * Add new pv_hashed_entry after header. @@ -877,13 +934,21 @@ dont_update_pte: } } } - + if (PV_HASHED_ENTRY_NULL == pvh_e) panic("Mapping alias chain exhaustion, possibly induced by numerous kernel virtual double mappings"); - pvh_e->va = vaddr; + pvh_e->va_and_flags = vaddr; pvh_e->pmap = pmap; pvh_e->ppn = pn; + if ((options & PMAP_OPTIONS_ALT_ACCT) && + IS_INTERNAL_PAGE(pai)) { + pvh_e->va_and_flags |= PVE_IS_ALTACCT; + is_altacct = TRUE; + } else { + pvh_e->va_and_flags &= ~PVE_IS_ALTACCT; + is_altacct = FALSE; + } pv_hash_add(pvh_e, pv_h); /* @@ -897,12 +962,12 @@ dont_update_pte: * for 'managed memory' */ pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE); - pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); OSAddAtomic(+1, &pmap->stats.resident_count); if (pmap->stats.resident_count > pmap->stats.resident_max) { pmap->stats.resident_max = pmap->stats.resident_count; } if (pmap != kernel_pmap) { + /* update pmap stats */ if (IS_REUSABLE_PAGE(pai)) { OSAddAtomic(+1, &pmap->stats.reusable); PMAP_STATS_PEAK(pmap->stats.reusable); @@ -913,13 +978,33 @@ dont_update_pte: OSAddAtomic(+1, &pmap->stats.external); PMAP_STATS_PEAK(pmap->stats.external); } + + /* update ledgers */ + if (is_altacct) { + /* internal but also alternate accounting */ + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + /* alternate accounting, so not in footprint */ + } else if (IS_REUSABLE_PAGE(pai)) { + assert(!is_altacct); + assert(IS_INTERNAL_PAGE(pai)); + /* internal but reusable: not in footprint */ + } else if (IS_INTERNAL_PAGE(pai)) { + assert(!is_altacct); + assert(!IS_REUSABLE_PAGE(pai)); + /* internal: add to footprint */ + pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } else { + /* not internal: not in footprint */ + } } } else if (last_managed_page == 0) { /* Account for early mappings created before "managed pages" * are determined. Consider consulting the available DRAM map. */ pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE); - pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); OSAddAtomic(+1, &pmap->stats.resident_count); if (pmap != kernel_pmap) { #if 00 @@ -1030,8 +1115,10 @@ Done: PMAP_ZINFO_PFREE(pmap, PAGE_SIZE); } - PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, 0, 0, 0, 0, 0); - return KERN_SUCCESS; + kr = KERN_SUCCESS; +done: + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr); + return kr; } /* @@ -1071,22 +1158,28 @@ pmap_remove_range_options( pv_hashed_entry_t pvh_e; int pvh_cnt = 0; int num_removed, num_unwired, num_found, num_invalid; - int num_device, num_external, num_internal, num_reusable; - uint64_t num_compressed; + int stats_external, stats_internal, stats_reusable; + uint64_t stats_compressed; + int ledgers_internal, ledgers_alt_internal; + uint64_t ledgers_compressed, ledgers_alt_compressed; ppnum_t pai; pmap_paddr_t pa; vm_map_offset_t vaddr; boolean_t is_ept = is_ept_pmap(pmap); + boolean_t was_altacct; num_removed = 0; num_unwired = 0; num_found = 0; num_invalid = 0; - num_device = 0; - num_external = 0; - num_internal = 0; - num_reusable = 0; - num_compressed = 0; + stats_external = 0; + stats_internal = 0; + stats_reusable = 0; + stats_compressed = 0; + ledgers_internal = 0; + ledgers_compressed = 0; + ledgers_alt_internal = 0; + ledgers_alt_compressed = 0; /* invalidate the PTEs first to "freeze" them */ for (cpte = spte, vaddr = start_vaddr; cpte < epte; @@ -1095,14 +1188,19 @@ pmap_remove_range_options( pa = pte_to_pa(p); if (pa == 0) { - if (pmap != kernel_pmap && - (options & PMAP_OPTIONS_REMOVE) && - (p & PTE_COMPRESSED)) { - /* one less "compressed" */ - num_compressed++; - /* clear marker */ + if ((options & PMAP_OPTIONS_REMOVE) && + (PTE_IS_COMPRESSED(p))) { + assert(pmap != kernel_pmap); + /* one less "compressed"... */ + stats_compressed++; + ledgers_compressed++; + if (p & PTE_COMPRESSED_ALT) { + /* ... but it used to be "ALTACCT" */ + ledgers_alt_compressed++; + } + /* clear marker(s) */ /* XXX probably does not need to be atomic! */ - pmap_update_pte(cpte, PTE_COMPRESSED, 0); + pmap_update_pte(cpte, INTEL_PTE_COMPRESSED_MASK, 0); } continue; } @@ -1119,7 +1217,6 @@ pmap_remove_range_options( * Just remove the mappings. */ pmap_store_pte(cpte, 0); - num_device++; continue; } @@ -1144,8 +1241,27 @@ pmap_remove_range_options( cpte++, vaddr += PAGE_SIZE_64) { pa = pte_to_pa(*cpte); - if (pa == 0) + if (pa == 0) { + check_pte_for_compressed_marker: + /* + * This PTE could have been replaced with a + * "compressed" marker after our first "freeze" + * loop above, so check again. + */ + if ((options & PMAP_OPTIONS_REMOVE) && + (PTE_IS_COMPRESSED(*cpte))) { + assert(pmap != kernel_pmap); + /* one less "compressed"... */ + stats_compressed++; + ledgers_compressed++; + if (*cpte & PTE_COMPRESSED_ALT) { + /* ... but it used to be "ALTACCT" */ + ledgers_alt_compressed++; + } + pmap_store_pte(cpte, 0); + } continue; + } pai = pa_index(pa); @@ -1154,15 +1270,40 @@ pmap_remove_range_options( pa = pte_to_pa(*cpte); if (pa == 0) { UNLOCK_PVH(pai); - continue; + goto check_pte_for_compressed_marker; } + + /* + * Remove the mapping from the pvlist for this physical page. + */ + pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte, &was_altacct); + num_removed++; + /* update pmap stats */ if (IS_REUSABLE_PAGE(pai)) { - num_reusable++; + stats_reusable++; + } else if (IS_INTERNAL_PAGE(pai)) { + stats_internal++; + } else { + stats_external++; + } + /* update ledgers */ + if (was_altacct) { + /* internal and alternate accounting */ + assert(IS_INTERNAL_PAGE(pai)); + ledgers_internal++; + ledgers_alt_internal++; + } else if (IS_REUSABLE_PAGE(pai)) { + /* internal but reusable */ + assert(!was_altacct); + assert(IS_INTERNAL_PAGE(pai)); } else if (IS_INTERNAL_PAGE(pai)) { - num_internal++; + /* internal */ + assert(!was_altacct); + assert(!IS_REUSABLE_PAGE(pai)); + ledgers_internal++; } else { - num_external++; + /* not internal */ } /* @@ -1170,13 +1311,13 @@ pmap_remove_range_options( * nuke the entry in the page table */ /* remember reference and change */ - pmap_phys_attributes[pai] |= - (char) (*cpte & (PHYS_MODIFIED | PHYS_REFERENCED)); - - /* - * Remove the mapping from the pvlist for this physical page. - */ - pvh_e = pmap_pv_remove(pmap, vaddr, (ppnum_t *) &pai, cpte); + if (!is_ept) { + pmap_phys_attributes[pai] |= + *cpte & (PHYS_MODIFIED | PHYS_REFERENCED); + } else { + pmap_phys_attributes[pai] |= + ept_refmod_to_physmap((*cpte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED); + } /* completely invalidate the PTE */ pmap_store_pte(cpte, 0); @@ -1206,35 +1347,72 @@ update_counts: panic("pmap_remove_range: resident_count"); #endif pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed)); - pmap_ledger_debit(pmap, task_ledgers.phys_footprint, machine_ptob(num_removed)); - assert(pmap->stats.resident_count >= num_removed); + PMAP_STATS_ASSERTF((pmap->stats.resident_count >= num_removed, + "pmap=%p num_removed=%d stats.resident_count=%d", + pmap, num_removed, pmap->stats.resident_count)); OSAddAtomic(-num_removed, &pmap->stats.resident_count); if (pmap != kernel_pmap) { -#if 00 - assert(pmap->stats.device >= num_device); - if (num_device) - OSAddAtomic(-num_device, &pmap->stats.device); -#endif /* 00 */ - assert(pmap->stats.external >= num_external); - if (num_external) - OSAddAtomic(-num_external, &pmap->stats.external); - assert(pmap->stats.internal >= num_internal); - if (num_internal) - OSAddAtomic(-num_internal, &pmap->stats.internal); - assert(pmap->stats.reusable >= num_reusable); - if (num_reusable) - OSAddAtomic(-num_reusable, &pmap->stats.reusable); - assert(pmap->stats.compressed >= num_compressed); - if (num_compressed) - OSAddAtomic64(-num_compressed, &pmap->stats.compressed); + PMAP_STATS_ASSERTF((pmap->stats.external >= stats_external, + "pmap=%p stats_external=%d stats.external=%d", + pmap, stats_external, pmap->stats.external)); + PMAP_STATS_ASSERTF((pmap->stats.internal >= stats_internal, + "pmap=%p stats_internal=%d stats.internal=%d", + pmap, stats_internal, pmap->stats.internal)); + PMAP_STATS_ASSERTF((pmap->stats.reusable >= stats_reusable, + "pmap=%p stats_reusable=%d stats.reusable=%d", + pmap, stats_reusable, pmap->stats.reusable)); + PMAP_STATS_ASSERTF((pmap->stats.compressed >= stats_compressed, + "pmap=%p stats_compressed=%lld, stats.compressed=%lld", + pmap, stats_compressed, pmap->stats.compressed)); + + /* update pmap stats */ + if (stats_external) { + OSAddAtomic(-stats_external, &pmap->stats.external); + } + if (stats_internal) { + OSAddAtomic(-stats_internal, &pmap->stats.internal); + } + if (stats_reusable) + OSAddAtomic(-stats_reusable, &pmap->stats.reusable); + if (stats_compressed) + OSAddAtomic64(-stats_compressed, &pmap->stats.compressed); + /* update ledgers */ + if (ledgers_internal) { + pmap_ledger_debit(pmap, + task_ledgers.internal, + machine_ptob(ledgers_internal)); + } + if (ledgers_compressed) { + pmap_ledger_debit(pmap, + task_ledgers.internal_compressed, + machine_ptob(ledgers_compressed)); + } + if (ledgers_alt_internal) { + pmap_ledger_debit(pmap, + task_ledgers.alternate_accounting, + machine_ptob(ledgers_alt_internal)); + } + if (ledgers_alt_compressed) { + pmap_ledger_debit(pmap, + task_ledgers.alternate_accounting_compressed, + machine_ptob(ledgers_alt_compressed)); + } + pmap_ledger_debit(pmap, + task_ledgers.phys_footprint, + machine_ptob((ledgers_internal - + ledgers_alt_internal) + + (ledgers_compressed - + ledgers_alt_compressed))); } #if TESTING if (pmap->stats.wired_count < num_unwired) panic("pmap_remove_range: wired_count"); #endif - assert(pmap->stats.wired_count >= num_unwired); + PMAP_STATS_ASSERTF((pmap->stats.wired_count >= num_unwired, + "pmap=%p num_unwired=%d stats.wired_count=%d", + pmap, num_unwired, pmap->stats.wired_count)); OSAddAtomic(-num_unwired, &pmap->stats.wired_count); pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired)); @@ -1279,10 +1457,8 @@ pmap_remove_options( is_ept = is_ept_pmap(map); PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START, - map, - (uint32_t) (s64 >> 32), s64, - (uint32_t) (e64 >> 32), e64); - + VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(s64), + VM_KERNEL_ADDRHIDE(e64)); PMAP_LOCK(map); @@ -1363,8 +1539,7 @@ pmap_remove_options( PMAP_UNLOCK(map); - PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END, - map, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END); } @@ -1417,8 +1592,8 @@ pmap_page_protect_options( */ return; } - PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, - pn, prot, 0, 0, 0); + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, pn, prot); /* * Determine the new protection. @@ -1461,7 +1636,7 @@ pmap_page_protect_options( pmap = pv_e->pmap; is_ept = is_ept_pmap(pmap); - vaddr = pv_e->va; + vaddr = PVE_VA(pv_e); pte = pmap_pte(pmap, vaddr); pmap_assert2((pa_index(pte_to_pa(*pte)) == pn), @@ -1488,8 +1663,12 @@ pmap_page_protect_options( if (pmap != kernel_pmap && (options & PMAP_OPTIONS_COMPRESSOR) && IS_INTERNAL_PAGE(pai)) { - /* mark this PTE as having been "reclaimed" */ + assert(!PTE_IS_COMPRESSED(*pte)); + /* mark this PTE as having been "compressed" */ new_pte_value = PTE_COMPRESSED; + if (IS_ALTACCT_PAGE(pai, pv_e)) { + new_pte_value |= PTE_COMPRESSED_ALT; + } } else { new_pte_value = 0; } @@ -1508,37 +1687,37 @@ pmap_page_protect_options( pmap_update_pte(pte, PTE_VALID_MASK(is_ept), 0); PMAP_UPDATE_TLBS(pmap, vaddr, vaddr+PAGE_SIZE); + if (!is_ept) { + pmap_phys_attributes[pai] |= + *pte & (PHYS_MODIFIED|PHYS_REFERENCED); + } else { + pmap_phys_attributes[pai] |= + ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED); + } if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED) && - ! (pmap_phys_attributes[pai] & - PHYS_MODIFIED) && - (*pte & PHYS_MODIFIED)) { + IS_INTERNAL_PAGE(pai) && + (pmap_phys_attributes[pai] & + PHYS_MODIFIED)) { /* * Page is actually "modified" and * will be compressed. Start * accounting for it as "compressed". */ + assert(!(options & PMAP_OPTIONS_COMPRESSOR)); options &= ~PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED; options |= PMAP_OPTIONS_COMPRESSOR; - new_pte_value = PTE_COMPRESSED; - } - if (!is_ept) { - pmap_phys_attributes[pai] |= - *pte & (PHYS_MODIFIED|PHYS_REFERENCED); - } else { - pmap_phys_attributes[pai] |= - ept_refmod_to_physmap((*pte & (INTEL_EPT_REF | INTEL_EPT_MOD))) & (PHYS_MODIFIED | PHYS_REFERENCED); + assert(new_pte_value == 0); + if (pmap != kernel_pmap) { + new_pte_value = PTE_COMPRESSED; + if (IS_ALTACCT_PAGE(pai, pv_e)) { + new_pte_value |= PTE_COMPRESSED_ALT; + } + } } pmap_store_pte(pte, new_pte_value); } - if (new_pte_value == PTE_COMPRESSED) { - /* one more "compressed" page */ - OSAddAtomic64(+1, &pmap->stats.compressed); - PMAP_STATS_PEAK(pmap->stats.compressed); - pmap->stats.compressed_lifetime++; - } - #if TESTING if (pmap->stats.resident_count < 1) panic("pmap_page_protect: resident_count"); @@ -1546,17 +1725,15 @@ pmap_page_protect_options( pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE); assert(pmap->stats.resident_count >= 1); OSAddAtomic(-1, &pmap->stats.resident_count); + + /* + * We only ever compress internal pages. + */ if (options & PMAP_OPTIONS_COMPRESSOR) { - /* - * This removal is only being done so we can send this page to - * the compressor; therefore it mustn't affect total task footprint. - */ - pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); - } else { - pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + assert(IS_INTERNAL_PAGE(pai)); } - if (pmap != kernel_pmap) { + /* update pmap stats */ if (IS_REUSABLE_PAGE(pai)) { assert(pmap->stats.reusable > 0); OSAddAtomic(-1, &pmap->stats.reusable); @@ -1567,6 +1744,59 @@ pmap_page_protect_options( assert(pmap->stats.external > 0); OSAddAtomic(-1, &pmap->stats.external); } + if ((options & PMAP_OPTIONS_COMPRESSOR) && + IS_INTERNAL_PAGE(pai)) { + /* adjust "compressed" stats */ + OSAddAtomic64(+1, &pmap->stats.compressed); + PMAP_STATS_PEAK(pmap->stats.compressed); + pmap->stats.compressed_lifetime++; + } + + /* update ledgers */ + if (IS_ALTACCT_PAGE(pai, pv_e)) { + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE); + } + } else if (IS_REUSABLE_PAGE(pai)) { + assert(!IS_ALTACCT_PAGE(pai, pv_e)); + assert(IS_INTERNAL_PAGE(pai)); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + /* was not in footprint, but is now */ + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } else if (IS_INTERNAL_PAGE(pai)) { + assert(!IS_ALTACCT_PAGE(pai, pv_e)); + assert(!IS_REUSABLE_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + /* + * Update all stats related to physical + * footprint, which only deals with + * internal pages. + */ + if (options & PMAP_OPTIONS_COMPRESSOR) { + /* + * This removal is only being + * done so we can send this page + * to the compressor; therefore + * it mustn't affect total task + * footprint. + */ + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + } else { + /* + * This internal page isn't + * going to the compressor, + * so adjust stats to keep + * phys_footprint up to date. + */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } } /* @@ -1621,7 +1851,7 @@ pmap_page_protect_options( if (pvh_e != (pv_hashed_entry_t) pv_h) { pv_hash_remove(pvh_e); pv_h->pmap = pvh_e->pmap; - pv_h->va = pvh_e->va; + pv_h->va_and_flags = pvh_e->va_and_flags; pvh_e->qlink.next = (queue_entry_t) pvh_eh; pvh_eh = pvh_e; @@ -1636,8 +1866,7 @@ pmap_page_protect_options( done: UNLOCK_PVH(pai); - PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END); } @@ -1653,11 +1882,11 @@ phys_attribute_clear( { pv_rooted_entry_t pv_h; pv_hashed_entry_t pv_e; - pt_entry_t *pte; + pt_entry_t *pte = NULL; int pai; pmap_t pmap; char attributes = 0; - boolean_t is_internal, is_reusable, is_ept; + boolean_t is_internal, is_reusable, is_altacct, is_ept; int ept_bits_to_clear; boolean_t ept_keep_global_mod = FALSE; @@ -1688,8 +1917,7 @@ phys_attribute_clear( return; } - PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, - pn, bits, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits); pv_h = pai_to_pvh(pai); @@ -1717,7 +1945,8 @@ phys_attribute_clear( pmap = pv_e->pmap; is_ept = is_ept_pmap(pmap); - va = pv_e->va; + is_altacct = IS_ALTACCT_PAGE(pai, pv_e); + va = PVE_VA(pv_e); pte_bits = 0; if (bits) { @@ -1781,10 +2010,23 @@ phys_attribute_clear( /* one more "internal" */ OSAddAtomic(+1, &pmap->stats.internal); PMAP_STATS_PEAK(pmap->stats.internal); + assert(pmap->stats.internal > 0); + if (is_altacct) { + /* no impact on ledgers */ + } else { + pmap_ledger_credit(pmap, + task_ledgers.internal, + PAGE_SIZE); + pmap_ledger_credit( + pmap, + task_ledgers.phys_footprint, + PAGE_SIZE); + } } else { /* one more "external" */ OSAddAtomic(+1, &pmap->stats.external); PMAP_STATS_PEAK(pmap->stats.external); + assert(pmap->stats.external > 0); } } else if ((options & PMAP_OPTIONS_SET_REUSABLE) && !is_reusable && @@ -1792,10 +2034,22 @@ phys_attribute_clear( /* one more "reusable" */ OSAddAtomic(+1, &pmap->stats.reusable); PMAP_STATS_PEAK(pmap->stats.reusable); + assert(pmap->stats.reusable > 0); if (is_internal) { /* one less "internal" */ assert(pmap->stats.internal > 0); OSAddAtomic(-1, &pmap->stats.internal); + if (is_altacct) { + /* no impact on footprint */ + } else { + pmap_ledger_debit(pmap, + task_ledgers.internal, + PAGE_SIZE); + pmap_ledger_debit( + pmap, + task_ledgers.phys_footprint, + PAGE_SIZE); + } } else { /* one less "external" */ assert(pmap->stats.external > 0); @@ -1833,8 +2087,7 @@ phys_attribute_clear( UNLOCK_PVH(pai); - PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END, - 0, 0, 0, 0, 0); + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END); } /* @@ -1900,7 +2153,7 @@ phys_attribute_test( pmap = pv_e->pmap; is_ept = is_ept_pmap(pmap); - va = pv_e->va; + va = PVE_VA(pv_e); /* * pick up modify and/or reference bits from mapping */ @@ -1942,7 +2195,8 @@ pmap_change_wiring( PMAP_LOCK(map); if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL) - panic("pmap_change_wiring: pte missing"); + panic("pmap_change_wiring(%p,0x%llx,%d): pte missing", + map, vaddr, wired); if (wired && !iswired(*pte)) { /* @@ -1980,9 +2234,11 @@ pmap_map_bd( unsigned int flags) { pt_entry_t template; - pt_entry_t *pte; - spl_t spl; + pt_entry_t *ptep; + vm_offset_t base = virt; + boolean_t doflush = FALSE; + template = pa_to_pte(start_addr) | INTEL_PTE_REF | INTEL_PTE_MOD @@ -1995,51 +2251,100 @@ pmap_map_bd( template |= INTEL_PTE_PTA; } -#if defined(__x86_64__) if ((prot & VM_PROT_EXECUTE) == 0) template |= INTEL_PTE_NX; -#endif if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; while (start_addr < end_addr) { - spl = splhigh(); - pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt); - if (pte == PT_ENTRY_NULL) { - panic("pmap_map_bd: Invalid kernel address\n"); + ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd: Invalid kernel address"); + } + if (pte_to_pa(*ptep)) { + doflush = TRUE; } - pmap_store_pte(pte, template); - splx(spl); + pmap_store_pte(ptep, template); pte_increment_pa(template); virt += PAGE_SIZE; start_addr += PAGE_SIZE; } - flush_tlb_raw(); - PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr); + if (doflush) { + flush_tlb_raw(); + PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr); + } return(virt); } -unsigned int +/* Create a virtual alias beginning at 'ava' of the specified kernel virtual + * range. The aliased pagetable range is expanded if + * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization, + * assumes caller has stabilized the source and destination ranges. Currently + * used to populate sections of the trampoline "doublemap" at CPU startup. + */ + +void +pmap_alias( + vm_offset_t ava, + vm_map_offset_t start_addr, + vm_map_offset_t end_addr, + vm_prot_t prot, + unsigned int eoptions) +{ + pt_entry_t prot_template, template; + pt_entry_t *aptep, *sptep; + + prot_template = INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID; + if ((prot & VM_PROT_EXECUTE) == 0) + prot_template |= INTEL_PTE_NX; + + if (prot & VM_PROT_WRITE) + prot_template |= INTEL_PTE_WRITE; + assert(((start_addr | end_addr) & PAGE_MASK) == 0); + while (start_addr < end_addr) { + aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava); + if (aptep == PT_ENTRY_NULL) { + if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) { + pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP); + aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava); + } else { + panic("pmap_alias: Invalid alias address"); + } + } + /* The aliased range should not have any active mappings */ + assert(pte_to_pa(*aptep) == 0); + + sptep = pmap_pte(kernel_pmap, start_addr); + assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0)); + template = pa_to_pte(pte_to_pa(*sptep)) | prot_template; + pmap_store_pte(aptep, template); + + ava += PAGE_SIZE; + start_addr += PAGE_SIZE; + } +} + +mach_vm_size_t pmap_query_resident( pmap_t pmap, addr64_t s64, addr64_t e64, - unsigned int *compressed_count_p) + mach_vm_size_t *compressed_bytes_p) { pt_entry_t *pde; pt_entry_t *spte, *epte; addr64_t l64; uint64_t deadline; - unsigned int result; + mach_vm_size_t resident_bytes; + mach_vm_size_t compressed_bytes; boolean_t is_ept; - unsigned int compressed_count; pmap_intr_assert(); if (pmap == PMAP_NULL || pmap == kernel_pmap || s64 == e64) { - if (compressed_count_p) { - *compressed_count_p = 0; + if (compressed_bytes_p) { + *compressed_bytes_p = 0; } return 0; } @@ -2047,12 +2352,11 @@ pmap_query_resident( is_ept = is_ept_pmap(pmap); PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START, - pmap, - (uint32_t) (s64 >> 32), s64, - (uint32_t) (e64 >> 32), e64); + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(s64), + VM_KERNEL_ADDRHIDE(e64)); - result = 0; - compressed_count = 0; + resident_bytes = 0; + compressed_bytes = 0; PMAP_LOCK(pmap); @@ -2075,9 +2379,9 @@ pmap_query_resident( for (; spte < epte; spte++) { if (pte_to_pa(*spte) != 0) { - result++; + resident_bytes += PAGE_SIZE; } else if (*spte & PTE_COMPRESSED) { - compressed_count++; + compressed_bytes += PAGE_SIZE; } } @@ -2095,20 +2399,96 @@ pmap_query_resident( PMAP_UNLOCK(pmap); PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END, - pmap, 0, 0, 0, 0); + resident_bytes); - if (compressed_count_p) { - *compressed_count_p = compressed_count; + if (compressed_bytes_p) { + *compressed_bytes_p = compressed_bytes; } - return result; + return resident_bytes; } -#if MACH_ASSERT -void -pmap_set_process( - __unused pmap_t pmap, - __unused int pid, - __unused char *procname) +kern_return_t +pmap_query_page_info( + pmap_t pmap, + vm_map_offset_t va, + int *disp_p) +{ + int disp; + boolean_t is_ept; + pmap_paddr_t pa; + ppnum_t pai; + pd_entry_t *pde; + pt_entry_t *pte; + + pmap_intr_assert(); + if (pmap == PMAP_NULL || pmap == kernel_pmap) { + *disp_p = 0; + return KERN_INVALID_ARGUMENT; + } + + disp = 0; + is_ept = is_ept_pmap(pmap); + + PMAP_LOCK(pmap); + + pde = pmap_pde(pmap, va); + if (!pde || + !(*pde & PTE_VALID_MASK(is_ept)) || + (*pde & PTE_PS)) { + goto done; + } + + pte = pmap_pte(pmap, va); + if (pte == PT_ENTRY_NULL) { + goto done; + } + + pa = pte_to_pa(*pte); + if (pa == 0) { + if (PTE_IS_COMPRESSED(*pte)) { + disp |= PMAP_QUERY_PAGE_COMPRESSED; + if (*pte & PTE_COMPRESSED_ALT) { + disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT; + } + } + } else { + disp |= PMAP_QUERY_PAGE_PRESENT; + pai = pa_index(pa); + if (!IS_MANAGED_PAGE(pai)) { + } else if (pmap_pv_is_altacct(pmap, va, pai)) { + assert(IS_INTERNAL_PAGE(pai)); + disp |= PMAP_QUERY_PAGE_INTERNAL; + disp |= PMAP_QUERY_PAGE_ALTACCT; + } else if (IS_REUSABLE_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_REUSABLE; + } else if (IS_INTERNAL_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_INTERNAL; + } + } + +done: + PMAP_UNLOCK(pmap); + *disp_p = disp; + return KERN_SUCCESS; +} + +void pmap_set_jit_entitled(__unused pmap_t pmap) +{ + /* The x86 pmap layer does not care if a map has a JIT entry. */ + return; +} + +bool pmap_has_prot_policy(__unused vm_prot_t prot) +{ + /* + * The x86 pmap layer does not apply any policy to any protection + * types. + */ + return FALSE; +} + +void pmap_release_pages_fast(void) { + return; } -#endif /* MACH_ASSERT */ +