#include <i386/tsc.h>
#include <i386/pmap_internal.h>
#include <i386/pmap_pcid.h>
+#if CONFIG_VMX
+#include <i386/vmx/vmx_cpu.h>
+#endif
#include <vm/vm_protos.h>
+#include <san/kasan.h>
#include <i386/mp.h>
#include <i386/mp_desc.h>
#include <pexpert/i386/efi.h>
+#if MACH_ASSERT
+int pmap_stats_assert = 1;
+#endif /* MACH_ASSERT */
#ifdef IWANTTODEBUG
#undef DEBUG
boolean_t no_shared_cr3 = DEBUG; /* TRUE for DEBUG by default */
-int nx_enabled = 1; /* enable no-execute protection */
+int nx_enabled = 1; /* enable no-execute protection -- set during boot */
+
+#if DEBUG || DEVELOPMENT
int allow_data_exec = VM_ABI_32; /* 32-bit apps may execute data by default, 64-bit apps may not */
int allow_stack_exec = 0; /* No apps may execute from the stack by default */
+#else /* DEBUG || DEVELOPMENT */
+const int allow_data_exec = VM_ABI_32; /* 32-bit apps may execute data by default, 64-bit apps may not */
+const int allow_stack_exec = 0; /* No apps may execute from the stack by default */
+#endif /* DEBUG || DEVELOPMENT */
const boolean_t cpu_64bit = TRUE; /* Mais oui! */
*/
boolean_t pmap_initialized = FALSE;/* Has pmap_init completed? */
-static struct vm_object kptobj_object_store;
-static struct vm_object kpml4obj_object_store;
-static struct vm_object kpdptobj_object_store;
+static struct vm_object kptobj_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+static struct vm_object kpml4obj_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+static struct vm_object kpdptobj_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
/*
* Array of physical page attribites for managed pages.
caddr_t DADDR1;
caddr_t DADDR2;
-const boolean_t pmap_disable_kheap_nx = FALSE;
-const boolean_t pmap_disable_kstack_nx = FALSE;
-extern boolean_t doconstro_override;
+boolean_t pmap_disable_kheap_nx = FALSE;
+boolean_t pmap_disable_kstack_nx = FALSE;
extern long __stack_chk_guard[];
+static uint64_t pmap_eptp_flags = 0;
+boolean_t pmap_ept_support_ad = FALSE;
+
+
/*
* Map memory at initialization. The physical addresses being
* mapped are not managed and are never unmapped.
vm_prot_t prot,
unsigned int flags)
{
+ kern_return_t kr;
int ps;
ps = PAGE_SIZE;
while (start_addr < end_addr) {
- pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
- (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+ kr = pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
+ (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+
+ if (kr != KERN_SUCCESS) {
+ panic("%s: failed pmap_enter, "
+ "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
+ __FUNCTION__,
+ (void *)virt, (void *)start_addr, (void *)end_addr, prot, flags);
+ }
+
virt += ps;
start_addr += ps;
}
extern vm_offset_t stext;
extern vm_offset_t etext;
extern vm_offset_t sdata, edata;
-extern vm_offset_t sconstdata, econstdata;
+extern vm_offset_t sconst, econst;
extern void *KPTphys;
cdp->cpu_task_map = TASK_MAP_64BIT;
pmap_pcid_configure();
if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
+ pmap_smep_enabled = TRUE;
+#if DEVELOPMENT || DEBUG
boolean_t nsmep;
- if (!PE_parse_boot_argn("-pmap_smep_disable", &nsmep, sizeof(nsmep))) {
+ if (PE_parse_boot_argn("-pmap_smep_disable", &nsmep, sizeof(nsmep))) {
+ pmap_smep_enabled = FALSE;
+ }
+#endif
+ if (pmap_smep_enabled) {
set_cr4(get_cr4() | CR4_SMEP);
- pmap_smep_enabled = TRUE;
+ }
+
+ }
+ if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMAP) {
+ pmap_smap_enabled = TRUE;
+#if DEVELOPMENT || DEBUG
+ boolean_t nsmap;
+ if (PE_parse_boot_argn("-pmap_smap_disable", &nsmap, sizeof(nsmap))) {
+ pmap_smap_enabled = FALSE;
+ }
+#endif
+ if (pmap_smap_enabled) {
+ set_cr4(get_cr4() | CR4_SMAP);
}
}
+#if !MONOTONIC
if (cdp->cpu_fixed_pmcs_enabled) {
boolean_t enable = TRUE;
cpu_pmc_control(&enable);
}
+#endif /* !MONOTONIC */
}
static uint32_t pmap_scale_shift(void) {
kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT);
kernel_pmap->pm_pml4 = IdlePML4;
kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
+ kernel_pmap->pm_eptp = 0;
pmap_pcid_initialize_kernel(kernel_pmap);
if (pmap_smep_enabled)
printf("PMAP: Supervisor Mode Execute Protection enabled\n");
+ if (pmap_smap_enabled)
+ printf("PMAP: Supervisor Mode Access Protection enabled\n");
#if DEBUG
printf("Stack canary: 0x%lx\n", __stack_chk_guard[0]);
printf("early_random(): 0x%qx\n", early_random());
#endif
+#if DEVELOPMENT || DEBUG
boolean_t ptmp;
/* Check if the user has requested disabling stack or heap no-execute
* enforcement. These are "const" variables; that qualifier is cast away
boolean_t *pdknhp = (boolean_t *) &pmap_disable_kstack_nx;
*pdknhp = TRUE;
}
+#endif /* DEVELOPMENT || DEBUG */
boot_args *args = (boot_args *)PE_state.bootArgs;
if (args->efiMode == kBootArgsEfiMode32) {
* in the DEBUG kernel) to force the kernel to switch to its own map
* (and cr3) when control is in kernelspace. The kernel's map does not
* include (i.e. share) userspace so wild references will cause
- * a panic. Only copyin and copyout are exempt from this.
+ * a panic. Only copyin and copyout are exempt from this.
*/
(void) PE_parse_boot_argn("-no_shared_cr3",
&no_shared_cr3, sizeof (no_shared_cr3));
if (no_shared_cr3)
kprintf("Kernel not sharing user map\n");
-
+
#ifdef PMAP_TRACES
if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
kprintf("Kernel traces for pmap operations enabled\n");
- }
+ }
#endif /* PMAP_TRACES */
+
+#if MACH_ASSERT
+ PE_parse_boot_argn("pmap_stats_assert",
+ &pmap_stats_assert,
+ sizeof (pmap_stats_assert));
+#endif /* MACH_ASSERT */
}
void
void
hibernate_rebuild_pmap_structs(void)
{
- int32_t cindx, eindx, rindx;
+ int32_t cindx, eindx, rindx = 0;
pv_rooted_entry_t pv_h;
eindx = (int32_t)pmap_npages;
+ pv_lock_table_size(npages)
+ pv_hash_lock_table_size((npvhashbuckets))
+ npages);
-
s = round_page(s);
if (kernel_memory_allocate(kernel_map, &addr, s, 0,
- KMA_KOBJECT | KMA_PERMANENT)
+ KMA_KOBJECT | KMA_PERMANENT, VM_KERN_MEMORY_PMAP)
!= KERN_SUCCESS)
panic("pmap_init");
pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,
4096 * 3 /* LCM x86_64*/, "pv_list");
zone_change(pv_hashed_list_zone, Z_NOENCRYPT, TRUE);
+ zone_change(pv_hashed_list_zone, Z_GZALLOC_EXEMPT, TRUE);
/* create pv entries for kernel pages mapped by low level
startup code. these have to exist so we can pmap_remove()
pv_rooted_entry_t pv_e;
pv_e = pai_to_pvh(ppn);
- pv_e->va = vaddr;
+ pv_e->va_and_flags = vaddr;
vaddr += PAGE_SIZE;
pv_e->pmap = kernel_pmap;
queue_init(&pv_e->qlink);
* before this is shared with any user.
*/
pmap_expand_pml4(kernel_pmap, KERNEL_BASEMENT, PMAP_EXPAND_OPTIONS_NONE);
+
+#if CONFIG_VMX
+ pmap_ept_support_ad = vmx_hv_support() && (VMX_CAP(MSR_IA32_VMX_EPT_VPID_CAP, MSR_IA32_VMX_EPT_VPID_CAP_AD_SHIFT, 1) ? TRUE : FALSE);
+ pmap_eptp_flags = HV_VMX_EPTP_MEMORY_TYPE_WB | HV_VMX_EPTP_WALK_LENGTH(4) | (pmap_ept_support_ad ? HV_VMX_EPTP_ENABLE_AD_FLAGS : 0);
+#endif /* CONFIG_VMX */
}
static
pd_entry_t *pdep;
pt_entry_t *ptep = NULL;
+ assert(!is_ept_pmap(npmap));
+
assert(((sv & 0xFFFULL) | (nxrosz & 0xFFFULL)) == 0);
for (pdep = pmap_pde(npmap, cv); pdep != NULL && (cv < ev);) {
* The now unused level-1 PTE pages are also freed.
*/
extern ppnum_t vm_kernel_base_page;
-void
-pmap_lowmem_finalize(void)
-{
+static uint32_t constptes = 0, dataptes = 0;
+
+void pmap_lowmem_finalize(void) {
spl_t spl;
int i;
}
boolean_t doconstro = TRUE;
-
+#if DEVELOPMENT || DEBUG
(void) PE_parse_boot_argn("dataconstro", &doconstro, sizeof(doconstro));
-
- if ((sconstdata | econstdata) & PAGE_MASK) {
- kprintf("Const DATA misaligned 0x%lx 0x%lx\n", sconstdata, econstdata);
- if ((sconstdata & PAGE_MASK) || (doconstro_override == FALSE))
- doconstro = FALSE;
- }
-
- if ((sconstdata > edata) || (sconstdata < sdata) || ((econstdata - sconstdata) >= (edata - sdata))) {
- kprintf("Const DATA incorrect size 0x%lx 0x%lx 0x%lx 0x%lx\n", sconstdata, econstdata, sdata, edata);
- doconstro = FALSE;
- }
-
- if (doconstro)
+#endif
+ if (doconstro) {
+ if (sconst & PAGE_MASK) {
+ panic("CONST segment misaligned 0x%lx 0x%lx\n",
+ sconst, econst);
+ }
kprintf("Marking const DATA read-only\n");
-
+ }
+
vm_offset_t dva;
for (dva = sdata; dva < edata; dva += I386_PGBYTES) {
assert(((sdata | edata) & PAGE_MASK) == 0);
- if ( (sdata | edata) & PAGE_MASK) {
- kprintf("DATA misaligned, 0x%lx, 0x%lx\n", sdata, edata);
- break;
- }
+ pt_entry_t dpte, *dptep = pmap_pte(kernel_pmap, dva);
+
+ dpte = *dptep;
+ assert((dpte & INTEL_PTE_VALID));
+ dpte |= INTEL_PTE_NX;
+ pmap_store_pte(dptep, dpte);
+ dataptes++;
+ }
+ assert(dataptes > 0);
+ for (dva = sconst; dva < econst; dva += I386_PGBYTES) {
pt_entry_t dpte, *dptep = pmap_pte(kernel_pmap, dva);
dpte = *dptep;
assert((dpte & INTEL_PTE_VALID));
- if ((dpte & INTEL_PTE_VALID) == 0) {
- kprintf("Missing data mapping 0x%lx 0x%lx 0x%lx\n", dva, sdata, edata);
- continue;
- }
-
dpte |= INTEL_PTE_NX;
- if (doconstro && (dva >= sconstdata) && (dva < econstdata)) {
- dpte &= ~INTEL_PTE_WRITE;
- }
+ dpte &= ~INTEL_PTE_WRITE;
+ constptes++;
pmap_store_pte(dptep, dpte);
}
+
+ assert(constptes > 0);
+
kernel_segment_command_t * seg;
kernel_section_t * sec;
return TRUE;
}
+void
+hv_ept_pmap_create(void **ept_pmap, void **eptp)
+{
+ pmap_t p;
+
+ if ((ept_pmap == NULL) || (eptp == NULL)) {
+ return;
+ }
+
+ p = pmap_create_options(get_task_ledger(current_task()), 0, (PMAP_CREATE_64BIT | PMAP_CREATE_EPT));
+ if (p == PMAP_NULL) {
+ *ept_pmap = NULL;
+ *eptp = NULL;
+ return;
+ }
+
+ assert(is_ept_pmap(p));
+
+ *ept_pmap = (void*)p;
+ *eptp = (void*)(p->pm_eptp);
+ return;
+}
/*
* Create and return a physical map.
* is bounded by that size.
*/
pmap_t
-pmap_create(
- ledger_t ledger,
- vm_map_size_t sz,
- boolean_t is_64bit)
+pmap_create_options(
+ ledger_t ledger,
+ vm_map_size_t sz,
+ int flags)
{
pmap_t p;
vm_size_t size;
pml4_entry_t *pml4;
pml4_entry_t *kpml4;
- PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
- (uint32_t) (sz>>32), (uint32_t) sz, is_64bit, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, sz, flags);
size = (vm_size_t) sz;
return(PMAP_NULL);
}
+ /*
+ * Return error when unrecognized flags are passed.
+ */
+ if ((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0) {
+ return(PMAP_NULL);
+ }
+
p = (pmap_t) zalloc(pmap_zone);
if (PMAP_NULL == p)
panic("pmap_create zalloc");
ledger_reference(ledger);
p->ledger = ledger;
- p->pm_task_map = is_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;;
- if (pmap_pcid_ncpus)
+ p->pm_task_map = ((flags & PMAP_CREATE_64BIT) ? TASK_MAP_64BIT : TASK_MAP_32BIT);
+
+ p->pagezero_accessible = FALSE;
+
+ if (pmap_pcid_ncpus) {
pmap_pcid_initialize(p);
+ }
p->pm_pml4 = zalloc(pmap_anchor_zone);
memset((char *)p->pm_pml4, 0, PAGE_SIZE);
- p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
+ if (flags & PMAP_CREATE_EPT) {
+ p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;
+ p->pm_cr3 = 0;
+ } else {
+ p->pm_eptp = 0;
+ p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
+ }
/* allocate the vm_objs to hold the pdpt, pde and pte pages */
if (NULL == p->pm_obj)
panic("pmap_create pte obj");
- /* All pmaps share the kernel's pml4 */
- pml4 = pmap64_pml4(p, 0ULL);
- kpml4 = kernel_pmap->pm_pml4;
- pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX];
- pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX];
- pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];
+ if (!(flags & PMAP_CREATE_EPT)) {
+ /* All host pmaps share the kernel's pml4 */
+ pml4 = pmap64_pml4(p, 0ULL);
+ kpml4 = kernel_pmap->pm_pml4;
+ pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX];
+ pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX];
+ pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];
+
+#if KASAN
+ pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0];
+ pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1];
+#endif
+ }
- PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
- p, is_64bit, 0, 0, 0);
+#if MACH_ASSERT
+ p->pmap_pid = 0;
+ strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
+#endif /* MACH_ASSERT */
+
+ PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END,
+ VM_KERNEL_ADDRHIDE(p));
return(p);
}
+pmap_t
+pmap_create(
+ ledger_t ledger,
+ vm_map_size_t sz,
+ boolean_t is_64bit)
+{
+ return pmap_create_options(ledger, sz, ((is_64bit) ? PMAP_CREATE_64BIT : 0));
+}
+
+/*
+ * We maintain stats and ledgers so that a task's physical footprint is:
+ * phys_footprint = ((internal - alternate_accounting)
+ * + (internal_compressed - alternate_accounting_compressed)
+ * + iokit_mapped
+ * + purgeable_nonvolatile
+ * + purgeable_nonvolatile_compressed
+ * + page_table)
+ * where "alternate_accounting" includes "iokit" and "purgeable" memory.
+ */
+
+#if MACH_ASSERT
+struct {
+ uint64_t num_pmaps_checked;
+
+ int phys_footprint_over;
+ ledger_amount_t phys_footprint_over_total;
+ ledger_amount_t phys_footprint_over_max;
+ int phys_footprint_under;
+ ledger_amount_t phys_footprint_under_total;
+ ledger_amount_t phys_footprint_under_max;
+
+ int internal_over;
+ ledger_amount_t internal_over_total;
+ ledger_amount_t internal_over_max;
+ int internal_under;
+ ledger_amount_t internal_under_total;
+ ledger_amount_t internal_under_max;
+
+ int internal_compressed_over;
+ ledger_amount_t internal_compressed_over_total;
+ ledger_amount_t internal_compressed_over_max;
+ int internal_compressed_under;
+ ledger_amount_t internal_compressed_under_total;
+ ledger_amount_t internal_compressed_under_max;
+
+ int iokit_mapped_over;
+ ledger_amount_t iokit_mapped_over_total;
+ ledger_amount_t iokit_mapped_over_max;
+ int iokit_mapped_under;
+ ledger_amount_t iokit_mapped_under_total;
+ ledger_amount_t iokit_mapped_under_max;
+
+ int alternate_accounting_over;
+ ledger_amount_t alternate_accounting_over_total;
+ ledger_amount_t alternate_accounting_over_max;
+ int alternate_accounting_under;
+ ledger_amount_t alternate_accounting_under_total;
+ ledger_amount_t alternate_accounting_under_max;
+
+ int alternate_accounting_compressed_over;
+ ledger_amount_t alternate_accounting_compressed_over_total;
+ ledger_amount_t alternate_accounting_compressed_over_max;
+ int alternate_accounting_compressed_under;
+ ledger_amount_t alternate_accounting_compressed_under_total;
+ ledger_amount_t alternate_accounting_compressed_under_max;
+
+ int page_table_over;
+ ledger_amount_t page_table_over_total;
+ ledger_amount_t page_table_over_max;
+ int page_table_under;
+ ledger_amount_t page_table_under_total;
+ ledger_amount_t page_table_under_max;
+
+ int purgeable_volatile_over;
+ ledger_amount_t purgeable_volatile_over_total;
+ ledger_amount_t purgeable_volatile_over_max;
+ int purgeable_volatile_under;
+ ledger_amount_t purgeable_volatile_under_total;
+ ledger_amount_t purgeable_volatile_under_max;
+
+ int purgeable_nonvolatile_over;
+ ledger_amount_t purgeable_nonvolatile_over_total;
+ ledger_amount_t purgeable_nonvolatile_over_max;
+ int purgeable_nonvolatile_under;
+ ledger_amount_t purgeable_nonvolatile_under_total;
+ ledger_amount_t purgeable_nonvolatile_under_max;
+
+ int purgeable_volatile_compressed_over;
+ ledger_amount_t purgeable_volatile_compressed_over_total;
+ ledger_amount_t purgeable_volatile_compressed_over_max;
+ int purgeable_volatile_compressed_under;
+ ledger_amount_t purgeable_volatile_compressed_under_total;
+ ledger_amount_t purgeable_volatile_compressed_under_max;
+
+ int purgeable_nonvolatile_compressed_over;
+ ledger_amount_t purgeable_nonvolatile_compressed_over_total;
+ ledger_amount_t purgeable_nonvolatile_compressed_over_max;
+ int purgeable_nonvolatile_compressed_under;
+ ledger_amount_t purgeable_nonvolatile_compressed_under_total;
+ ledger_amount_t purgeable_nonvolatile_compressed_under_max;
+} pmap_ledgers_drift;
+static void pmap_check_ledgers(pmap_t pmap);
+#else /* MACH_ASSERT */
+static inline void pmap_check_ledgers(__unused pmap_t pmap) {}
+#endif /* MACH_ASSERT */
+
/*
* Retire the given physical map from service.
* Should only be called if the map contains
* no valid mappings.
*/
+extern int vm_wired_objects_page_count;
void
pmap_destroy(pmap_t p)
return;
PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
- p, 0, 0, 0, 0);
+ VM_KERNEL_ADDRHIDe(p));
PMAP_LOCK(p);
PMAP_UNLOCK(p);
if (c != 0) {
- PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
- p, 1, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
pmap_assert(p == kernel_pmap);
return; /* still in use */
}
OSAddAtomic(-inuse_ptepages, &inuse_ptepages_count);
PMAP_ZINFO_PFREE(p, inuse_ptepages * PAGE_SIZE);
+
+ pmap_check_ledgers(p);
ledger_dereference(p->ledger);
zfree(pmap_zone, p);
- PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
}
/*
vm_map_offset_t orig_sva;
boolean_t set_NX;
int num_found = 0;
+ boolean_t is_ept;
pmap_intr_assert();
pmap_remove_options(map, sva, eva, options);
return;
}
+
PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
- map,
- (uint32_t) (sva >> 32), (uint32_t) sva,
- (uint32_t) (eva >> 32), (uint32_t) eva);
+ VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(sva),
+ VM_KERNEL_ADDRHIDE(eva));
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled)
set_NX = FALSE;
else
set_NX = TRUE;
+ is_ept = is_ept_pmap(map);
+
+
PMAP_LOCK(map);
orig_sva = sva;
if (lva > eva)
lva = eva;
pde = pmap_pde(map, sva);
- if (pde && (*pde & INTEL_PTE_VALID)) {
- if (*pde & INTEL_PTE_PS) {
+ if (pde && (*pde & PTE_VALID_MASK(is_ept))) {
+ if (*pde & PTE_PS) {
/* superpage */
spte = pde;
epte = spte+1; /* excluded */
}
for (; spte < epte; spte++) {
- if (!(*spte & INTEL_PTE_VALID))
+ if (!(*spte & PTE_VALID_MASK(is_ept)))
continue;
+ if (is_ept) {
+ if (prot & VM_PROT_READ)
+ pmap_update_pte(spte, 0, PTE_READ(is_ept));
+ else
+ pmap_update_pte(spte, PTE_READ(is_ept), 0);
+ }
if (prot & VM_PROT_WRITE)
- pmap_update_pte(spte, 0, INTEL_PTE_WRITE);
+ pmap_update_pte(spte, 0, PTE_WRITE(is_ept));
else
- pmap_update_pte(spte, INTEL_PTE_WRITE, 0);
+ pmap_update_pte(spte, PTE_WRITE(is_ept), 0);
- if (set_NX)
- pmap_update_pte(spte, 0, INTEL_PTE_NX);
- else
- pmap_update_pte(spte, INTEL_PTE_NX, 0);
+ if (set_NX) {
+ if (!is_ept)
+ pmap_update_pte(spte, 0, INTEL_PTE_NX);
+ else
+ pmap_update_pte(spte, INTEL_EPT_EX, 0);
+ } else {
+ if (!is_ept)
+ pmap_update_pte(spte, INTEL_PTE_NX, 0);
+ else
+ pmap_update_pte(spte, 0, INTEL_EPT_EX);
+ }
num_found++;
}
}
}
PMAP_UNLOCK(map);
- PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);
}
/* Map a (possibly) autogenned block */
-void
+kern_return_t
pmap_map_block(
- pmap_t pmap,
+ pmap_t pmap,
addr64_t va,
ppnum_t pa,
uint32_t size,
int attr,
__unused unsigned int flags)
{
+ kern_return_t kr;
+ addr64_t original_va = va;
uint32_t page;
int cur_page_size;
if (attr & VM_MEM_SUPERPAGE)
cur_page_size = SUPERPAGE_SIZE;
- else
+ else
cur_page_size = PAGE_SIZE;
for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) {
- pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+ kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+
+ if (kr != KERN_SUCCESS) {
+ /*
+ * This will panic for now, as it is unclear that
+ * removing the mappings is correct.
+ */
+ panic("%s: failed pmap_enter, "
+ "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
+ __FUNCTION__,
+ pmap, va, pa, size, prot, flags);
+
+ pmap_remove(pmap, original_va, va - original_va);
+ return kr;
+ }
+
va += cur_page_size;
pa+=cur_page_size/PAGE_SIZE;
}
+
+ return KERN_SUCCESS;
}
kern_return_t
uint64_t i;
ppnum_t pn;
pml4_entry_t *pml4p;
+ boolean_t is_ept = is_ept_pmap(map);
DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
* put the page into the pmap's obj list so it
* can be found later.
*/
- pn = m->phys_page;
+ pn = VM_PAGE_GET_PHYS_PAGE(m);
pa = i386_ptob(pn);
i = pml4idx(map, vaddr);
pmap_zero_page(pn);
vm_page_lockspin_queues();
- vm_page_wire(m);
+ vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
OSAddAtomic(1, &inuse_ptepages_count);
map, map->pm_obj_pml4, vaddr, i);
}
#endif
- vm_page_insert(m, map->pm_obj_pml4, (vm_object_offset_t)i * PAGE_SIZE);
+ vm_page_insert_wired(m, map->pm_obj_pml4, (vm_object_offset_t)i * PAGE_SIZE, VM_KERN_MEMORY_PTE);
vm_object_unlock(map->pm_obj_pml4);
/*
pml4p = pmap64_pml4(map, vaddr); /* refetch under lock */
pmap_store_pte(pml4p, pa_to_pte(pa)
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
PMAP_UNLOCK(map);
uint64_t i;
ppnum_t pn;
pdpt_entry_t *pdptp;
+ boolean_t is_ept = is_ept_pmap(map);
DBG("pmap_expand_pdpt(%p,%p)\n", map, (void *)vaddr);
* put the page into the pmap's obj list so it
* can be found later.
*/
- pn = m->phys_page;
+ pn = VM_PAGE_GET_PHYS_PAGE(m);
pa = i386_ptob(pn);
i = pdptidx(map, vaddr);
pmap_zero_page(pn);
vm_page_lockspin_queues();
- vm_page_wire(m);
+ vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
OSAddAtomic(1, &inuse_ptepages_count);
map, map->pm_obj_pdpt, vaddr, i);
}
#endif
- vm_page_insert(m, map->pm_obj_pdpt, (vm_object_offset_t)i * PAGE_SIZE);
+ vm_page_insert_wired(m, map->pm_obj_pdpt, (vm_object_offset_t)i * PAGE_SIZE, VM_KERN_MEMORY_PTE);
vm_object_unlock(map->pm_obj_pdpt);
/*
pdptp = pmap64_pdpt(map, vaddr); /* refetch under lock */
pmap_store_pte(pdptp, pa_to_pte(pa)
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
PMAP_UNLOCK(map);
unsigned int options)
{
pt_entry_t *pdp;
- register vm_page_t m;
- register pmap_paddr_t pa;
+ vm_page_t m;
+ pmap_paddr_t pa;
uint64_t i;
ppnum_t pn;
+ boolean_t is_ept = is_ept_pmap(map);
/*
* put the page into the pmap's obj list so it
* can be found later.
*/
- pn = m->phys_page;
+ pn = VM_PAGE_GET_PHYS_PAGE(m);
pa = i386_ptob(pn);
i = pdeidx(map, vaddr);
pmap_zero_page(pn);
vm_page_lockspin_queues();
- vm_page_wire(m);
+ vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
OSAddAtomic(1, &inuse_ptepages_count);
map, map->pm_obj, vaddr, i);
}
#endif
- vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i * PAGE_SIZE);
+ vm_page_insert_wired(m, map->pm_obj, (vm_object_offset_t)i * PAGE_SIZE, VM_KERN_MEMORY_PTE);
vm_object_unlock(map->pm_obj);
/*
*/
pdp = pmap_pde(map, vaddr);
pmap_store_pte(pdp, pa_to_pte(pa)
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
PMAP_UNLOCK(map);
{
ppnum_t pn;
pt_entry_t *pte;
+ boolean_t is_ept = is_ept_pmap(pmap);
PMAP_LOCK(pmap);
pte = pmap64_pml4(pmap, vaddr);
pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
}
if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {
pte = pmap64_pdpt(pmap, vaddr);
pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
}
if(pmap_pte(pmap, vaddr) == PT_ENTRY_NULL) {
pte = pmap64_pde(pmap, vaddr);
pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
- | INTEL_PTE_VALID
- | INTEL_PTE_USER
- | INTEL_PTE_WRITE);
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
}
PMAP_UNLOCK(pmap);
pmap_collect(
pmap_t p)
{
- register pt_entry_t *pdp, *ptp;
+ pt_entry_t *pdp, *ptp;
pt_entry_t *eptp;
int wired;
+ boolean_t is_ept;
if (p == PMAP_NULL)
return;
if (p == kernel_pmap)
return;
+ is_ept = is_ept_pmap(p);
+
/*
* Garbage collect map.
*/
pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
pdp++)
{
- if (*pdp & INTEL_PTE_VALID) {
- if(*pdp & INTEL_PTE_REF) {
- pmap_store_pte(pdp, *pdp & ~INTEL_PTE_REF);
- collect_ref++;
- } else {
- collect_unref++;
- ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
- eptp = ptp + NPTEPG;
+ if (*pdp & PTE_VALID_MASK(is_ept)) {
+ if (*pdp & PTE_REF(is_ept)) {
+ pmap_store_pte(pdp, *pdp & ~PTE_REF(is_ept));
+ collect_ref++;
+ } else {
+ collect_unref++;
+ ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
+ eptp = ptp + NPTEPG;
- /*
- * If the pte page has any wired mappings, we cannot
- * free it.
- */
- wired = 0;
- {
- register pt_entry_t *ptep;
- for (ptep = ptp; ptep < eptp; ptep++) {
- if (iswired(*ptep)) {
- wired = 1;
- break;
+ /*
+ * If the pte page has any wired mappings, we cannot
+ * free it.
+ */
+ wired = 0;
+ {
+ pt_entry_t *ptep;
+ for (ptep = ptp; ptep < eptp; ptep++) {
+ if (iswired(*ptep)) {
+ wired = 1;
+ break;
+ }
+ }
+ }
+ if (!wired) {
+ /*
+ * Remove the virtual addresses mapped by this pte page.
+ */
+ pmap_remove_range(p,
+ pdetova(pdp - (pt_entry_t *)p->dirbase),
+ ptp,
+ eptp);
+
+ /*
+ * Invalidate the page directory pointer.
+ */
+ pmap_store_pte(pdp, 0x0);
+
+ PMAP_UNLOCK(p);
+
+ /*
+ * And free the pte page itself.
+ */
+ {
+ vm_page_t m;
+
+ vm_object_lock(p->pm_obj);
+
+ m = vm_page_lookup(p->pm_obj,(vm_object_offset_t)(pdp - (pt_entry_t *)&p->dirbase[0]) * PAGE_SIZE);
+ if (m == VM_PAGE_NULL)
+ panic("pmap_collect: pte page not in object");
+
+ vm_object_unlock(p->pm_obj);
+
+ VM_PAGE_FREE(m);
+
+ OSAddAtomic(-1, &inuse_ptepages_count);
+ PMAP_ZINFO_PFREE(p, PAGE_SIZE);
+ }
+
+ PMAP_LOCK(p);
+ }
}
- }
- }
- if (!wired) {
- /*
- * Remove the virtual addresses mapped by this pte page.
- */
- pmap_remove_range(p,
- pdetova(pdp - (pt_entry_t *)p->dirbase),
- ptp,
- eptp);
-
- /*
- * Invalidate the page directory pointer.
- */
- pmap_store_pte(pdp, 0x0);
-
- PMAP_UNLOCK(p);
-
- /*
- * And free the pte page itself.
- */
- {
- register vm_page_t m;
-
- vm_object_lock(p->pm_obj);
-
- m = vm_page_lookup(p->pm_obj,(vm_object_offset_t)(pdp - (pt_entry_t *)&p->dirbase[0]) * PAGE_SIZE);
- if (m == VM_PAGE_NULL)
- panic("pmap_collect: pte page not in object");
-
- vm_object_unlock(p->pm_obj);
-
- VM_PAGE_FREE(m);
-
- OSAddAtomic(-1, &inuse_ptepages_count);
- PMAP_ZINFO_PFREE(p, PAGE_SIZE);
- }
-
- PMAP_LOCK(p);
}
- }
- }
}
PMAP_UPDATE_TLBS(p, 0x0, 0xFFFFFFFFFFFFF000ULL);
PMAP_UNLOCK(p);
return;
-
}
#endif
#endif /* MACH_VM_DEBUG */
-
+#if CONFIG_COREDUMP
/* temporary workaround */
boolean_t
coredumpok(__unused vm_map_t map, __unused vm_offset_t va)
return TRUE;
#endif
}
-
+#endif
boolean_t
phys_page_exists(ppnum_t pn)
pfc->pfc_invalid_global = 0;
}
-extern unsigned TLBTimeOut;
+extern uint64_t TLBTimeOut;
void
pmap_flush(
pmap_flush_context *pfc)
cpus_to_signal = pfc->pfc_cpus;
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_START,
- NULL, cpus_to_signal, 0, 0, 0);
+ NULL, cpus_to_signal);
for (cpu = 0, cpu_bit = 1; cpu < real_ncpus && cpus_to_signal; cpu++, cpu_bit <<= 1) {
deadline = mach_absolute_time() +
(TLBTimeOut ? TLBTimeOut : LockTimeOut);
boolean_t is_timeout_traced = FALSE;
-
+
/*
* Wait for those other cpus to acknowledge
*/
if (TLBTimeOut == 0) {
if (is_timeout_traced)
continue;
+
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
- NULL, cpus_to_signal, cpus_to_respond, 0, 0);
+ NULL, cpus_to_signal, cpus_to_respond);
+
is_timeout_traced = TRUE;
continue;
}
- pmap_tlb_flush_timeout = TRUE;
orig_acks = NMIPI_acks;
- mp_cpus_NMIPI(cpus_to_respond);
-
- panic("TLB invalidation IPI timeout: "
- "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%lx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
- cpus_to_respond, orig_acks, NMIPI_acks);
+ NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+ panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+ cpus_to_respond, orig_acks, NMIPI_acks, deadline);
}
}
}
+
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_END,
- NULL, cpus_signaled, flush_self, 0, 0);
+ NULL, cpus_signaled, flush_self);
mp_enable_preemption();
}
+static void
+invept(void *eptp)
+{
+ struct {
+ uint64_t eptp;
+ uint64_t reserved;
+ } __attribute__((aligned(16), packed)) invept_descriptor = {(uint64_t)eptp, 0};
+
+ __asm__ volatile("invept (%%rax), %%rcx"
+ : : "c" (PMAP_INVEPT_SINGLE_CONTEXT), "a" (&invept_descriptor)
+ : "cc", "memory");
+}
+
/*
* Called with pmap locked, we:
* - scan through per-cpu data to see which other cpus need to flush
{
unsigned int cpu;
unsigned int cpu_bit;
- cpumask_t cpus_to_signal;
+ cpumask_t cpus_to_signal = 0;
unsigned int my_cpu = cpu_number();
pmap_paddr_t pmap_cr3 = pmap->pm_cr3;
boolean_t flush_self = FALSE;
boolean_t pmap_is_shared = (pmap->pm_shared || (pmap == kernel_pmap));
boolean_t need_global_flush = FALSE;
uint32_t event_code;
+ vm_map_offset_t event_startv, event_endv;
+ boolean_t is_ept = is_ept_pmap(pmap);
assert((processor_avail_count < 2) ||
(ml_get_interrupts_enabled() && get_preemption_level() != 0));
- event_code = (pmap == kernel_pmap) ? PMAP_CODE(PMAP__FLUSH_KERN_TLBS)
- : PMAP_CODE(PMAP__FLUSH_TLBS);
+ if (pmap == kernel_pmap) {
+ event_code = PMAP_CODE(PMAP__FLUSH_KERN_TLBS);
+ event_startv = VM_KERNEL_UNSLIDE_OR_PERM(startv);
+ event_endv = VM_KERNEL_UNSLIDE_OR_PERM(endv);
+ } else if (is_ept) {
+ event_code = PMAP_CODE(PMAP__FLUSH_EPT);
+ event_startv = startv;
+ event_endv = endv;
+ } else {
+ event_code = PMAP_CODE(PMAP__FLUSH_TLBS);
+ event_startv = startv;
+ event_endv = endv;
+ }
+
PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_START,
- pmap, options, startv, endv, 0);
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap), options,
+ event_startv, event_endv);
+
+ if (is_ept) {
+ mp_cpus_call(CPUMASK_ALL, ASYNC, invept, (void*)pmap->pm_eptp);
+ goto out;
+ }
/*
* Scan other cpus for matching active or task CR3.
* For idle cpus (with no active map) we mark them invalid but
* don't signal -- they'll check as they go busy.
*/
- cpus_to_signal = 0;
-
if (pmap_pcid_ncpus) {
if (pmap_is_shared)
need_global_flush = TRUE;
/* cut tracepoint but don't panic */
if (is_timeout_traced)
continue;
- PMAP_TRACE_CONSTANT(
- PMAP_CODE(PMAP__FLUSH_TLBS_TO),
- pmap, cpus_to_signal, cpus_to_respond, 0, 0);
+
+ PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap),
+ cpus_to_signal,
+ cpus_to_respond);
+
is_timeout_traced = TRUE;
continue;
}
- pmap_tlb_flush_timeout = TRUE;
orig_acks = NMIPI_acks;
- mp_cpus_NMIPI(cpus_to_respond);
- panic("TLB invalidation IPI timeout: "
- "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%lx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
- cpus_to_respond, orig_acks, NMIPI_acks);
+ NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+ panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+ cpus_to_respond, orig_acks, NMIPI_acks, deadline);
}
}
}
out:
PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_END,
- pmap, cpus_to_signal, startv, endv, 0);
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal,
+ event_startv, event_endv);
}
void
pmap_update_interrupt(void)
{
- PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START);
if (current_cpu_datap()->cpu_tlb_invalid)
process_pmap_updates();
- PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END);
}
#include <mach/mach_vm.h> /* mach_vm_region_recurse() */
kern_return_t rv = KERN_SUCCESS;
uint64_t skip4 = 0, skip2 = 0;
+ assert(!is_ept_pmap(ipmap));
+
sv &= ~PAGE_MASK_64;
ev &= ~PAGE_MASK_64;
while (cv < ev) {
}
return rv;
}
+
+#if MACH_ASSERT
+extern int pmap_ledgers_panic;
+static void
+pmap_check_ledgers(
+ pmap_t pmap)
+{
+ ledger_amount_t bal;
+ int pid;
+ char *procname;
+ boolean_t do_panic;
+
+ if (pmap->pmap_pid == 0) {
+ /*
+ * This pmap was not or is no longer fully associated
+ * with a task (e.g. the old pmap after a fork()/exec() or
+ * spawn()). Its "ledger" still points at a task that is
+ * now using a different (and active) address space, so
+ * we can't check that all the pmap ledgers are balanced here.
+ *
+ * If the "pid" is set, that means that we went through
+ * pmap_set_process() in task_terminate_internal(), so
+ * this task's ledger should not have been re-used and
+ * all the pmap ledgers should be back to 0.
+ */
+ return;
+ }
+
+ do_panic = FALSE;
+ pid = pmap->pmap_pid;
+ procname = pmap->pmap_procname;
+
+ pmap_ledgers_drift.num_pmaps_checked++;
+
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.phys_footprint,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"phys_footprint\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.phys_footprint_over++;
+ pmap_ledgers_drift.phys_footprint_over_total += bal;
+ if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
+ pmap_ledgers_drift.phys_footprint_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.phys_footprint_under++;
+ pmap_ledgers_drift.phys_footprint_under_total += bal;
+ if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
+ pmap_ledgers_drift.phys_footprint_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.internal,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"internal\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.internal_over++;
+ pmap_ledgers_drift.internal_over_total += bal;
+ if (bal > pmap_ledgers_drift.internal_over_max) {
+ pmap_ledgers_drift.internal_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.internal_under++;
+ pmap_ledgers_drift.internal_under_total += bal;
+ if (bal < pmap_ledgers_drift.internal_under_max) {
+ pmap_ledgers_drift.internal_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.internal_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"internal_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.internal_compressed_over++;
+ pmap_ledgers_drift.internal_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
+ pmap_ledgers_drift.internal_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.internal_compressed_under++;
+ pmap_ledgers_drift.internal_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
+ pmap_ledgers_drift.internal_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.iokit_mapped,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"iokit_mapped\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.iokit_mapped_over++;
+ pmap_ledgers_drift.iokit_mapped_over_total += bal;
+ if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
+ pmap_ledgers_drift.iokit_mapped_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.iokit_mapped_under++;
+ pmap_ledgers_drift.iokit_mapped_under_total += bal;
+ if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
+ pmap_ledgers_drift.iokit_mapped_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.alternate_accounting,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"alternate_accounting\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.alternate_accounting_over++;
+ pmap_ledgers_drift.alternate_accounting_over_total += bal;
+ if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
+ pmap_ledgers_drift.alternate_accounting_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.alternate_accounting_under++;
+ pmap_ledgers_drift.alternate_accounting_under_total += bal;
+ if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
+ pmap_ledgers_drift.alternate_accounting_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.alternate_accounting_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"alternate_accounting_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.alternate_accounting_compressed_over++;
+ pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
+ pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.alternate_accounting_compressed_under++;
+ pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
+ pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.page_table,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"page_table\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.page_table_over++;
+ pmap_ledgers_drift.page_table_over_total += bal;
+ if (bal > pmap_ledgers_drift.page_table_over_max) {
+ pmap_ledgers_drift.page_table_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.page_table_under++;
+ pmap_ledgers_drift.page_table_under_total += bal;
+ if (bal < pmap_ledgers_drift.page_table_under_max) {
+ pmap_ledgers_drift.page_table_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_volatile,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_volatile\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_volatile_over++;
+ pmap_ledgers_drift.purgeable_volatile_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
+ pmap_ledgers_drift.purgeable_volatile_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_volatile_under++;
+ pmap_ledgers_drift.purgeable_volatile_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
+ pmap_ledgers_drift.purgeable_volatile_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_nonvolatile,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_nonvolatile\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_nonvolatile_over++;
+ pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_nonvolatile_under++;
+ pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_volatile_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_volatile_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_over++;
+ pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_volatile_compressed_under++;
+ pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_nonvolatile_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_nonvolatile_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
+ }
+ }
+ }
+
+ if (do_panic) {
+ if (pmap_ledgers_panic) {
+ panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
+ pmap, pid, procname);
+ } else {
+ printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
+ pmap, pid, procname);
+ }
+ }
+
+ if (pmap->stats.resident_count != 0 ||
+ pmap->stats.wired_count != 0 ||
+ pmap->stats.device != 0 ||
+ pmap->stats.internal != 0 ||
+ pmap->stats.external != 0 ||
+ pmap->stats.reusable != 0 ||
+ pmap->stats.compressed != 0) {
+ if (pmap_stats_assert) {
+ panic("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
+ pmap, pid, procname,
+ pmap->stats.resident_count,
+ pmap->stats.wired_count,
+ pmap->stats.device,
+ pmap->stats.internal,
+ pmap->stats.external,
+ pmap->stats.reusable,
+ pmap->stats.compressed);
+ } else {
+ printf("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
+ pmap, pid, procname,
+ pmap->stats.resident_count,
+ pmap->stats.wired_count,
+ pmap->stats.device,
+ pmap->stats.internal,
+ pmap->stats.external,
+ pmap->stats.reusable,
+ pmap->stats.compressed);
+ }
+ }
+}
+
+void
+pmap_set_process(
+ pmap_t pmap,
+ int pid,
+ char *procname)
+{
+ if (pmap == NULL)
+ return;
+
+ pmap->pmap_pid = pid;
+ strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
+}
+#endif /* MACH_ASSERT */
+
+
+#if DEVELOPMENT || DEBUG
+int pmap_pagezero_mitigation = 1;
+#endif
+
+void pmap_advise_pagezero_range(pmap_t lpmap, uint64_t low_bound) {
+#if DEVELOPMENT || DEBUG
+ if (pmap_pagezero_mitigation == 0) {
+ lpmap->pagezero_accessible = FALSE;
+ return;
+ }
+#endif
+ lpmap->pagezero_accessible = ((pmap_smap_enabled == FALSE) && (low_bound < 0x1000));
+ if (lpmap == current_pmap()) {
+ mp_disable_preemption();
+ current_cpu_datap()->cpu_pagezero_mapped = lpmap->pagezero_accessible;
+ mp_enable_preemption();
+ }
+}
+
+void pmap_verify_noncacheable(uintptr_t vaddr) {
+ pt_entry_t *ptep = NULL;
+ ptep = pmap_pte(kernel_pmap, vaddr);
+ if (ptep == NULL) {
+ panic("pmap_verify_noncacheable: no translation for 0x%lx", vaddr);
+ }
+ /* Non-cacheable OK */
+ if (*ptep & (INTEL_PTE_NCACHE))
+ return;
+ /* Write-combined OK */
+ if (*ptep & (INTEL_PTE_PTA))
+ return;
+ panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
+}
+
+#if KASAN
+void kasan_map_low_fixed_regions(void) {
+ kasan_map_shadow(MASTER_GDT_ALIAS, PAGE_SIZE, false);
+ kasan_map_shadow(MASTER_IDT_ALIAS, PAGE_SIZE, false);
+ kasan_map_shadow(LOWGLOBAL_ALIAS, PAGE_SIZE, false);
+}
+#endif
projects / apple / xnu.git / blobdiff