X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..d26ffc64f583ab2d29df48f13518685602bc8832:/osfmk/i386/pmap_internal.h diff --git a/osfmk/i386/pmap_internal.h b/osfmk/i386/pmap_internal.h index 6227e50f7..4ddabaa20 100644 --- a/osfmk/i386/pmap_internal.h +++ b/osfmk/i386/pmap_internal.h @@ -63,16 +63,15 @@ #ifdef PMAP_TRACES extern boolean_t pmap_trace; -#define PMAP_TRACE(x,a,b,c,d,e) \ - if (pmap_trace) { \ - KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e); \ +#define PMAP_TRACE(...) \ + if (pmap_trace) { \ + KDBG_RELEASE(__VA_ARGS__); \ } #else -#define PMAP_TRACE(x,a,b,c,d,e) KERNEL_DEBUG(x,a,b,c,d,e) +#define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__) #endif /* PMAP_TRACES */ -#define PMAP_TRACE_CONSTANT(x,a,b,c,d,e) \ - KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e); \ +#define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__) kern_return_t pmap_expand_pml4( pmap_t map, @@ -219,7 +218,7 @@ than the original pv lists that contained all aliases for the specific ppn. typedef struct pv_rooted_entry { /* first three entries must match pv_hashed_entry_t */ queue_head_t qlink; - vm_map_offset_t va; /* virtual address for mapping */ + vm_map_offset_t va_and_flags; /* virtual address for mapping */ pmap_t pmap; /* pmap where mapping lies */ } *pv_rooted_entry_t; @@ -228,7 +227,7 @@ typedef struct pv_rooted_entry { typedef struct pv_hashed_entry { /* first three entries must match pv_rooted_entry_t */ queue_head_t qlink; - vm_map_offset_t va; + vm_map_offset_t va_and_flags; pmap_t pmap; ppnum_t ppn; struct pv_hashed_entry *nexth; @@ -236,6 +235,12 @@ typedef struct pv_hashed_entry { #define PV_HASHED_ENTRY_NULL ((pv_hashed_entry_t)0) +#define PVE_VA(pve) ((pve)->va_and_flags & ~PAGE_MASK) +#define PVE_FLAGS(pve) ((pve)->va_and_flags & PAGE_MASK) +#define PVE_IS_ALTACCT 0x001 +#define PVE_IS_ALTACCT_PAGE(pve) \ + (((pve)->va_and_flags & PVE_IS_ALTACCT) ? TRUE : FALSE) + //#define PV_DEBUG 1 /* uncomment to enable some PV debugging code */ #ifdef PV_DEBUG #define CHK_NPVHASH() if(0 == npvhashmask) panic("npvhash uninitialized"); @@ -379,27 +384,30 @@ static inline void pmap_pv_throttle(__unused pmap_t p) { (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_INTERNAL)) #define IS_REUSABLE_PAGE(x) \ (IS_MANAGED_PAGE(x) && (pmap_phys_attributes[x] & PHYS_REUSABLE)) +#define IS_ALTACCT_PAGE(x,pve) \ + (IS_MANAGED_PAGE((x)) && \ + (PVE_IS_ALTACCT_PAGE((pve)))) /* * Physical page attributes. Copy bits from PTE definition. */ #define PHYS_MODIFIED INTEL_PTE_MOD /* page modified */ #define PHYS_REFERENCED INTEL_PTE_REF /* page referenced */ -#define PHYS_MANAGED INTEL_PTE_VALID /* page is managed */ -#define PHYS_NOENCRYPT INTEL_PTE_USER /* no need to encrypt this page in the hibernation image */ +#define PHYS_MANAGED INTEL_PTE_VALID /* page is managed */ +#define PHYS_NOENCRYPT INTEL_PTE_USER /* no need to encrypt this page in the hibernation image */ #define PHYS_NCACHE INTEL_PTE_NCACHE #define PHYS_PTA INTEL_PTE_PTA #define PHYS_CACHEABILITY_MASK (INTEL_PTE_PTA | INTEL_PTE_NCACHE) -#define PHYS_INTERNAL INTEL_PTE_WTHRU /* page from internal object */ -#define PHYS_REUSABLE INTEL_PTE_WRITE /* page is "reusable" */ +#define PHYS_INTERNAL INTEL_PTE_WTHRU /* page from internal object */ +#define PHYS_REUSABLE INTEL_PTE_WRITE /* page is "reusable" */ -extern const boolean_t pmap_disable_kheap_nx; -extern const boolean_t pmap_disable_kstack_nx; +extern boolean_t pmap_disable_kheap_nx; +extern boolean_t pmap_disable_kstack_nx; #define PMAP_EXPAND_OPTIONS_NONE (0x0) #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT) #define PMAP_EXPAND_OPTIONS_NOENTER (PMAP_OPTIONS_NOENTER) - +#define PMAP_EXPAND_OPTIONS_ALIASMAP (0x40000000U) /* * Amount of virtual memory mapped by one * page-directory entry. @@ -465,11 +473,14 @@ extern ppnum_t highest_hi; #define MAX_PREEMPTION_LATENCY_NS 20000 extern uint64_t max_preemption_latency_tsc; -/* #define DEBUGINTERRUPTS 1 uncomment to ensure pmap callers have interrupts enabled */ -#ifdef DEBUGINTERRUPTS +#if DEBUG +#define PMAP_INTR_DEBUG (1) +#endif + +#if PMAP_INTR_DEBUG #define pmap_intr_assert() { \ if (processor_avail_count > 1 && !ml_get_interrupts_enabled()) \ - panic("pmap interrupt assert %s, %d",__FILE__, __LINE__); \ + panic("pmap interrupt assert %d %s, %d", processor_avail_count, __FILE__, __LINE__); \ } #else #define pmap_intr_assert() @@ -487,7 +498,6 @@ pvhashidx(pmap_t pmap, vm_map_offset_t va) return hashidx; } - /* * unlinks the pv_hashed_entry_t pvh from the singly linked hash chain. * properly deals with the anchor. @@ -501,7 +511,7 @@ pmap_pvh_unlink(pv_hashed_entry_t pvh) int pvhash_idx; CHK_NPVHASH(); - pvhash_idx = pvhashidx(pvh->pmap, pvh->va); + pvhash_idx = pvhashidx(pvh->pmap, PVE_VA(pvh)); pprevh = pvhash(pvhash_idx); @@ -530,7 +540,7 @@ pv_hash_add(pv_hashed_entry_t pvh_e, int pvhash_idx; CHK_NPVHASH(); - pvhash_idx = pvhashidx(pvh_e->pmap, pvh_e->va); + pvhash_idx = pvhashidx(pvh_e->pmap, PVE_VA(pvh_e)); LOCK_PV_HASH(pvhash_idx); insque(&pvh_e->qlink, &pv_h->qlink); hashp = pvhash(pvhash_idx); @@ -549,7 +559,7 @@ pv_hash_remove(pv_hashed_entry_t pvh_e) int pvhash_idx; CHK_NPVHASH(); - pvhash_idx = pvhashidx(pvh_e->pmap,pvh_e->va); + pvhash_idx = pvhashidx(pvh_e->pmap,PVE_VA(pvh_e)); LOCK_PV_HASH(pvhash_idx); remque(&pvh_e->qlink); pmap_pvh_unlink(pvh_e); @@ -651,8 +661,10 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t * pv_rooted_entry_t pv_e = pv_h; uint32_t bitdex; pmap_t pvpmap = pv_h->pmap; - vm_map_offset_t pvva = pv_h->va; + vm_map_offset_t pvva = PVE_VA(pv_h); + vm_map_offset_t pve_flags; boolean_t ppcd = FALSE; + boolean_t is_ept; /* Ideally, we'd consult the Mach VM here to definitively determine * the nature of the mapping for this address space and address. @@ -664,16 +676,18 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t * /* As a precautionary measure, mark A+D */ pmap_phys_attributes[ppn_to_pai(ppn)] |= (PHYS_MODIFIED | PHYS_REFERENCED); + is_ept = is_ept_pmap(pmap); /* * Correct potential single bit errors in either (but not both) element * of the PV */ do { - if ((popcnt1((uintptr_t)pv_e->pmap ^ (uintptr_t)pmap) && pv_e->va == vaddr) || - (pv_e->pmap == pmap && popcnt1(pv_e->va ^ vaddr))) { + if ((popcnt1((uintptr_t)pv_e->pmap ^ (uintptr_t)pmap) && PVE_VA(pv_e) == vaddr) || + (pv_e->pmap == pmap && popcnt1(PVE_VA(pv_e) ^ vaddr))) { + pve_flags = PVE_FLAGS(pv_e); pv_e->pmap = pmap; - pv_e->va = vaddr; + pv_h->va_and_flags = vaddr | pve_flags; suppress_reason = PV_BITFLIP; action = PMAP_ACTION_RETRY; goto pmap_cpc_exit; @@ -688,7 +702,7 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t * ppnum_t npn = cpn ^ (ppnum_t) (1ULL << bitdex); if (IS_MANAGED_PAGE(npn)) { pv_rooted_entry_t npv_h = pai_to_pvh(ppn_to_pai(npn)); - if (npv_h->va == vaddr && npv_h->pmap == pmap) { + if (PVE_VA(npv_h) == vaddr && npv_h->pmap == pmap) { suppress_reason = PTE_BITFLIP; suppress_ppn = npn; action = PMAP_ACTION_RETRY_RELOCK; @@ -704,9 +718,11 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t * goto pmap_cpc_exit; } - /* Check for malformed/inconsistent entries */ - - if ((cpte & (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU | INTEL_PTE_PTA)) == (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU)) { + /* + * Check for malformed/inconsistent entries. + * The first check here isn't useful for EPT PTEs because INTEL_EPT_NCACHE == 0 + */ + if (!is_ept && ((cpte & (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU | INTEL_PTE_PTA)) == (INTEL_PTE_NCACHE | INTEL_PTE_WTHRU))) { action = PMAP_ACTION_IGNORE; suppress_reason = PTE_INVALID_CACHEABILITY; } @@ -714,7 +730,7 @@ pmap_classify_pagetable_corruption(pmap_t pmap, vm_map_offset_t vaddr, ppnum_t * action = PMAP_ACTION_IGNORE; suppress_reason = PTE_RSVD; } - else if ((pmap != kernel_pmap) && ((cpte & INTEL_PTE_USER) == 0)) { + else if ((pmap != kernel_pmap) && (!is_ept) && ((cpte & INTEL_PTE_USER) == 0)) { action = PMAP_ACTION_IGNORE; suppress_reason = PTE_SUPERVISOR; } @@ -745,8 +761,9 @@ pmap_cpc_exit: static inline __attribute__((always_inline)) pv_hashed_entry_t pmap_pv_remove(pmap_t pmap, vm_map_offset_t vaddr, - ppnum_t *ppnp, - pt_entry_t *pte) + ppnum_t *ppnp, + pt_entry_t *pte, + boolean_t *was_altacct) { pv_hashed_entry_t pvh_e; pv_rooted_entry_t pv_h; @@ -755,6 +772,7 @@ pmap_pv_remove(pmap_t pmap, uint32_t pv_cnt; ppnum_t ppn; + *was_altacct = FALSE; pmap_pv_remove_retry: ppn = *ppnp; pvh_e = PV_HASHED_ENTRY_NULL; @@ -765,7 +783,7 @@ pmap_pv_remove_retry: if (pac == PMAP_ACTION_IGNORE) goto pmap_pv_remove_exit; else if (pac == PMAP_ACTION_ASSERT) - panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p, %p): null pv_list!", pmap, vaddr, ppn, *pte, ppnp, pte); + panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p, %p): null pv_list, priors: %d", pmap, vaddr, ppn, *pte, ppnp, pte, pmap_pagetable_corruption_incidents); else if (pac == PMAP_ACTION_RETRY_RELOCK) { LOCK_PVH(ppn_to_pai(*ppnp)); pmap_phys_attributes[ppn_to_pai(*ppnp)] |= (PHYS_MODIFIED | PHYS_REFERENCED); @@ -775,7 +793,8 @@ pmap_pv_remove_retry: goto pmap_pv_remove_retry; } - if (pv_h->va == vaddr && pv_h->pmap == pmap) { + if (PVE_VA(pv_h) == vaddr && pv_h->pmap == pmap) { + *was_altacct = IS_ALTACCT_PAGE(ppn_to_pai(*ppnp), pv_h); /* * Header is the pv_rooted_entry. * We can't free that. If there is a queued @@ -790,19 +809,20 @@ pmap_pv_remove_retry: * and install as new root. */ CHK_NPVHASH(); - pvhash_idx = pvhashidx(pvh_e->pmap, pvh_e->va); + pvhash_idx = pvhashidx(pvh_e->pmap, PVE_VA(pvh_e)); LOCK_PV_HASH(pvhash_idx); remque(&pvh_e->qlink); pprevh = pvhash(pvhash_idx); if (PV_HASHED_ENTRY_NULL == *pprevh) { panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x): " - "empty hash, removing rooted", - pmap, vaddr, ppn); + "empty hash, removing rooted, priors: %d", + pmap, vaddr, ppn, pmap_pagetable_corruption_incidents); } pmap_pvh_unlink(pvh_e); UNLOCK_PV_HASH(pvhash_idx); pv_h->pmap = pvh_e->pmap; - pv_h->va = pvh_e->va; /* dispose of pvh_e */ + pv_h->va_and_flags = pvh_e->va_and_flags; + /* dispose of pvh_e */ } else { /* none queued after rooted */ pv_h->pmap = PMAP_NULL; @@ -818,8 +838,8 @@ pmap_pv_remove_retry: LOCK_PV_HASH(pvhash_idx); pprevh = pvhash(pvhash_idx); if (PV_HASHED_ENTRY_NULL == *pprevh) { - panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p): empty hash", - pmap, vaddr, ppn, *pte, pte); + panic("Possible memory corruption: pmap_pv_remove(%p,0x%llx,0x%x, 0x%llx, %p): empty hash, priors: %d", + pmap, vaddr, ppn, *pte, pte, pmap_pagetable_corruption_incidents); } pvh_e = *pprevh; pmap_pv_hashlist_walks++; @@ -827,7 +847,7 @@ pmap_pv_remove_retry: while (PV_HASHED_ENTRY_NULL != pvh_e) { pv_cnt++; if (pvh_e->pmap == pmap && - pvh_e->va == vaddr && + PVE_VA(pvh_e) == vaddr && pvh_e->ppn == ppn) break; pprevh = &pvh_e->nexth; @@ -838,7 +858,7 @@ pmap_pv_remove_retry: pmap_pagetable_corruption_action_t pac = pmap_classify_pagetable_corruption(pmap, vaddr, ppnp, pte, ROOT_PRESENT); if (pac == PMAP_ACTION_ASSERT) - panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, pv_h->va); + panic("Possible memory corruption: pmap_pv_remove(%p, 0x%llx, 0x%x, 0x%llx, %p, %p): pv not on hash, head: %p, 0x%llx, priors: %d", pmap, vaddr, ppn, *pte, ppnp, pte, pv_h->pmap, PVE_VA(pv_h), pmap_pagetable_corruption_incidents); else { UNLOCK_PV_HASH(pvhash_idx); if (pac == PMAP_ACTION_RETRY_RELOCK) { @@ -855,6 +875,8 @@ pmap_pv_remove_retry: } } + *was_altacct = IS_ALTACCT_PAGE(ppn_to_pai(*ppnp), pvh_e); + pmap_pv_hashlist_cnts += pv_cnt; if (pmap_pv_hashlist_max < pv_cnt) pmap_pv_hashlist_max = pv_cnt; @@ -866,34 +888,67 @@ pmap_pv_remove_exit: return pvh_e; } +static inline __attribute__((always_inline)) boolean_t +pmap_pv_is_altacct( + pmap_t pmap, + vm_map_offset_t vaddr, + ppnum_t ppn) +{ + pv_hashed_entry_t pvh_e; + pv_rooted_entry_t pv_h; + int pvhash_idx; + boolean_t is_altacct; + + pvh_e = PV_HASHED_ENTRY_NULL; + pv_h = pai_to_pvh(ppn_to_pai(ppn)); + + if (__improbable(pv_h->pmap == PMAP_NULL)) { + return FALSE; + } + + if (PVE_VA(pv_h) == vaddr && pv_h->pmap == pmap) { + /* + * Header is the pv_rooted_entry. + */ + return IS_ALTACCT_PAGE(ppn, pv_h); + } + + CHK_NPVHASH(); + pvhash_idx = pvhashidx(pmap, vaddr); + LOCK_PV_HASH(pvhash_idx); + pvh_e = *(pvhash(pvhash_idx)); + if (PV_HASHED_ENTRY_NULL == pvh_e) { + panic("Possible memory corruption: pmap_pv_is_altacct(%p,0x%llx,0x%x): empty hash", + pmap, vaddr, ppn); + } + while (PV_HASHED_ENTRY_NULL != pvh_e) { + if (pvh_e->pmap == pmap && + PVE_VA(pvh_e) == vaddr && + pvh_e->ppn == ppn) + break; + pvh_e = pvh_e->nexth; + } + if (PV_HASHED_ENTRY_NULL == pvh_e) { + is_altacct = FALSE; + } else { + is_altacct = IS_ALTACCT_PAGE(ppn, pvh_e); + } + UNLOCK_PV_HASH(pvhash_idx); + + return is_altacct; +} extern int pt_fake_zone_index; static inline void PMAP_ZINFO_PALLOC(pmap_t pmap, vm_size_t bytes) { - thread_t thr = current_thread(); - task_t task; - zinfo_usage_t zinfo; - pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes); - - if (pt_fake_zone_index != -1 && - (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL) - OSAddAtomic64(bytes, (int64_t *)&zinfo[pt_fake_zone_index].alloc); } static inline void PMAP_ZINFO_PFREE(pmap_t pmap, vm_size_t bytes) { - thread_t thr = current_thread(); - task_t task; - zinfo_usage_t zinfo; - pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes); - - if (pt_fake_zone_index != -1 && - (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL) - OSAddAtomic64(bytes, (int64_t *)&zinfo[pt_fake_zone_index].free); } static inline void @@ -911,11 +966,6 @@ PMAP_ZINFO_SFREE(pmap_t pmap, vm_size_t bytes) extern boolean_t pmap_initialized;/* Has pmap_init completed? */ #define valid_page(x) (pmap_initialized && pmap_valid_page(x)) -// XXX -#define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */ -// XXX - - int phys_attribute_test( ppnum_t phys, int bits); @@ -980,7 +1030,6 @@ static inline void pmap_update_pte(pt_entry_t *mptep, uint64_t pclear_bits, uint } while (!pmap_cmpx_pte(mptep, opte, npte)); } -#if defined(__x86_64__) /* * The single pml4 page per pmap is allocated at pmap create time and exists * for the duration of the pmap. we allocate this page in kernel vm. @@ -995,13 +1044,28 @@ pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr) return (NULL); } -#if PMAP_ASSERT +#if DEBUG return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_cr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]); #else return &pmap->pm_pml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)]; #endif } +static inline pml4_entry_t * +pmap64_user_pml4(pmap_t pmap, vm_map_offset_t vaddr) +{ + if (__improbable((vaddr > 0x00007FFFFFFFFFFFULL) && + (vaddr < 0xFFFF800000000000ULL))) { + return (NULL); + } + +#if DEBUG + return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_ucr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]); +#else + return &pmap->pm_upml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)]; +#endif +} + /* * Returns address of requested PDPT entry in the physmap. */ @@ -1010,9 +1074,12 @@ pmap64_pdpt(pmap_t pmap, vm_map_offset_t vaddr) { pml4_entry_t newpf; pml4_entry_t *pml4; + boolean_t is_ept; pml4 = pmap64_pml4(pmap, vaddr); - if (pml4 && ((*pml4 & INTEL_PTE_VALID))) { + is_ept = is_ept_pmap(pmap); + + if (pml4 && (*pml4 & PTE_VALID_MASK(is_ept))) { newpf = *pml4 & PG_FRAME; return &((pdpt_entry_t *) PHYSMAP_PTOV(newpf)) [(vaddr >> PDPTSHIFT) & (NPDPTPG-1)]; @@ -1027,10 +1094,12 @@ pmap64_pde(pmap_t pmap, vm_map_offset_t vaddr) { pdpt_entry_t newpf; pdpt_entry_t *pdpt; + boolean_t is_ept; pdpt = pmap64_pdpt(pmap, vaddr); + is_ept = is_ept_pmap(pmap); - if (pdpt && ((*pdpt & INTEL_PTE_VALID))) { + if (pdpt && (*pdpt & PTE_VALID_MASK(is_ept))) { newpf = *pdpt & PG_FRAME; return &((pd_entry_t *) PHYSMAP_PTOV(newpf)) [(vaddr >> PDSHIFT) & (NPDPG-1)]; @@ -1060,12 +1129,15 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr) { pd_entry_t *pde; pd_entry_t newpf; + boolean_t is_ept; assert(pmap); pde = pmap64_pde(pmap, vaddr); - if (pde && ((*pde & INTEL_PTE_VALID))) { - if (*pde & INTEL_PTE_PS) + is_ept = is_ept_pmap(pmap); + + if (pde && (*pde & PTE_VALID_MASK(is_ept))) { + if (*pde & PTE_PS) return pde; newpf = *pde & PG_FRAME; return &((pt_entry_t *)PHYSMAP_PTOV(newpf)) @@ -1073,7 +1145,13 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr) } return (NULL); } -#endif +extern void pmap_alias( + vm_offset_t ava, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t prot, + unsigned int options); + #if DEBUG #define DPRINTF(x...) kprintf(x) #else