#endif
#include <vm/vm_protos.h>
+#include <san/kasan.h>
#include <i386/mp.h>
#include <i386/mp_desc.h>
struct zone *pmap_zone; /* zone of pmap structures */
struct zone *pmap_anchor_zone;
+struct zone *pmap_uanchor_zone;
int pmap_debug = 0; /* flag for debugging prints */
unsigned int inuse_ptepages_count = 0;
vm_prot_t prot,
unsigned int flags)
{
+ kern_return_t kr;
int ps;
ps = PAGE_SIZE;
while (start_addr < end_addr) {
- pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
- (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+ kr = pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
+ (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+
+ if (kr != KERN_SUCCESS) {
+ panic("%s: failed pmap_enter, "
+ "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
+ __FUNCTION__,
+ (void *)virt, (void *)start_addr, (void *)end_addr, prot, flags);
+ }
+
virt += ps;
start_addr += ps;
}
pmap_cpu_init(void)
{
cpu_data_t *cdp = current_cpu_datap();
- /*
- * Here early in the life of a processor (from cpu_mode_init()).
- * Ensure global page feature is disabled at this point.
- */
- set_cr4(get_cr4() &~ CR4_PGE);
+ set_cr4(get_cr4() | CR4_PGE);
/*
* Initialize the per-cpu, TLB-related fields.
*/
cdp->cpu_kernel_cr3 = kernel_pmap->pm_cr3;
+ cpu_shadowp(cdp->cpu_number)->cpu_kernel_cr3 = cdp->cpu_kernel_cr3;
cdp->cpu_active_cr3 = kernel_pmap->pm_cr3;
cdp->cpu_tlb_invalid = FALSE;
cdp->cpu_task_map = TASK_MAP_64BIT;
+
pmap_pcid_configure();
if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
pmap_smep_enabled = TRUE;
}
}
+#if !MONOTONIC
if (cdp->cpu_fixed_pmcs_enabled) {
boolean_t enable = TRUE;
cpu_pmc_control(&enable);
}
+#endif /* !MONOTONIC */
}
static uint32_t pmap_scale_shift(void) {
kernel_pmap->nx_enabled = TRUE;
kernel_pmap->pm_task_map = TASK_MAP_64BIT;
kernel_pmap->pm_obj = (vm_object_t) NULL;
- kernel_pmap->dirbase = (pd_entry_t *)((uintptr_t)IdlePTD);
- kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT);
kernel_pmap->pm_pml4 = IdlePML4;
+ kernel_pmap->pm_upml4 = IdlePML4;
kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
+ kernel_pmap->pm_ucr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
kernel_pmap->pm_eptp = 0;
- pmap_pcid_initialize_kernel(kernel_pmap);
-
+ pmap_pcid_initialize_kernel(kernel_pmap);
- current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
+ current_cpu_datap()->cpu_kernel_cr3 = cpu_shadowp(cpu_number())->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
nkpt = NKPT;
OSAddAtomic(NKPT, &inuse_ptepages_count);
printf("Stack canary: 0x%lx\n", __stack_chk_guard[0]);
printf("early_random(): 0x%qx\n", early_random());
#endif
+#if DEVELOPMENT || DEBUG
boolean_t ptmp;
/* Check if the user has requested disabling stack or heap no-execute
* enforcement. These are "const" variables; that qualifier is cast away
boolean_t *pdknhp = (boolean_t *) &pmap_disable_kstack_nx;
*pdknhp = TRUE;
}
+#endif /* DEVELOPMENT || DEBUG */
boot_args *args = (boot_args *)PE_state.bootArgs;
if (args->efiMode == kBootArgsEfiMode32) {
* in the DEBUG kernel) to force the kernel to switch to its own map
* (and cr3) when control is in kernelspace. The kernel's map does not
* include (i.e. share) userspace so wild references will cause
- * a panic. Only copyin and copyout are exempt from this.
+ * a panic. Only copyin and copyout are exempt from this.
*/
(void) PE_parse_boot_argn("-no_shared_cr3",
&no_shared_cr3, sizeof (no_shared_cr3));
if (no_shared_cr3)
kprintf("Kernel not sharing user map\n");
-
+
#ifdef PMAP_TRACES
if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
kprintf("Kernel traces for pmap operations enabled\n");
- }
+ }
#endif /* PMAP_TRACES */
#if MACH_ASSERT
*/
zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);
+/* TODO: possible general optimisation...pre-allocate via zones commonly created
+ * level3/2 pagetables
+ */
+ pmap_uanchor_zone = zinit(PAGE_SIZE, task_max, PAGE_SIZE, "pagetable user anchors");
+ zone_change(pmap_uanchor_zone, Z_NOENCRYPT, TRUE);
+
+ /* The anchor is required to be page aligned. Zone debugging adds
+ * padding which may violate that requirement. Tell the zone
+ * subsystem that alignment is required.
+ */
+
+ zone_change(pmap_uanchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);
s = (vm_size_t) sizeof(struct pv_hashed_entry);
pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,
4096 * 3 /* LCM x86_64*/, "pv_list");
zone_change(pv_hashed_list_zone, Z_NOENCRYPT, TRUE);
+ zone_change(pv_hashed_list_zone, Z_GZALLOC_EXEMPT, TRUE);
/* create pv entries for kernel pages mapped by low level
startup code. these have to exist so we can pmap_remove()
* the map will be used in software only, and
* is bounded by that size.
*/
+
pmap_t
pmap_create_options(
ledger_t ledger,
pml4_entry_t *pml4;
pml4_entry_t *kpml4;
- PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
- (uint32_t) (sz>>32), (uint32_t) sz, flags, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, sz, flags);
size = (vm_size_t) sz;
/*
* Return error when unrecognized flags are passed.
*/
- if ((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0) {
+ if (__improbable((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0)) {
return(PMAP_NULL);
}
p = (pmap_t) zalloc(pmap_zone);
if (PMAP_NULL == p)
panic("pmap_create zalloc");
+
/* Zero all fields */
bzero(p, sizeof(*p));
/* init counts now since we'll be bumping some */
simple_lock_init(&p->lock, 0);
-#if 00
- p->stats.resident_count = 0;
- p->stats.resident_max = 0;
- p->stats.wired_count = 0;
-#else
bzero(&p->stats, sizeof (p->stats));
-#endif
+
p->ref_count = 1;
p->nx_enabled = 1;
p->pm_shared = FALSE;
}
p->pm_pml4 = zalloc(pmap_anchor_zone);
+ p->pm_upml4 = zalloc(pmap_uanchor_zone); //cleanup for EPT
pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0);
+ pmap_assert((((uintptr_t)p->pm_upml4) & PAGE_MASK) == 0);
memset((char *)p->pm_pml4, 0, PAGE_SIZE);
+ memset((char *)p->pm_upml4, 0, PAGE_SIZE);
if (flags & PMAP_CREATE_EPT) {
p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;
} else {
p->pm_eptp = 0;
p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
+ p->pm_ucr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_upml4);
}
/* allocate the vm_objs to hold the pdpt, pde and pte pages */
pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX];
pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX];
pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];
+ pml4[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];
+#if KASAN
+ pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0];
+ pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1];
+#endif
+ pml4_entry_t *pml4u = pmap64_user_pml4(p, 0ULL);
+ pml4u[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];
}
#if MACH_ASSERT
+ p->pmap_stats_assert = TRUE;
p->pmap_pid = 0;
strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
#endif /* MACH_ASSERT */
- PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
- p, flags, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END,
+ VM_KERNEL_ADDRHIDE(p));
return(p);
}
int purgeable_nonvolatile_compressed_under;
ledger_amount_t purgeable_nonvolatile_compressed_under_total;
ledger_amount_t purgeable_nonvolatile_compressed_under_max;
+
+ int network_volatile_over;
+ ledger_amount_t network_volatile_over_total;
+ ledger_amount_t network_volatile_over_max;
+ int network_volatile_under;
+ ledger_amount_t network_volatile_under_total;
+ ledger_amount_t network_volatile_under_max;
+
+ int network_nonvolatile_over;
+ ledger_amount_t network_nonvolatile_over_total;
+ ledger_amount_t network_nonvolatile_over_max;
+ int network_nonvolatile_under;
+ ledger_amount_t network_nonvolatile_under_total;
+ ledger_amount_t network_nonvolatile_under_max;
+
+ int network_volatile_compressed_over;
+ ledger_amount_t network_volatile_compressed_over_total;
+ ledger_amount_t network_volatile_compressed_over_max;
+ int network_volatile_compressed_under;
+ ledger_amount_t network_volatile_compressed_under_total;
+ ledger_amount_t network_volatile_compressed_under_max;
+
+ int network_nonvolatile_compressed_over;
+ ledger_amount_t network_nonvolatile_compressed_over_total;
+ ledger_amount_t network_nonvolatile_compressed_over_max;
+ int network_nonvolatile_compressed_under;
+ ledger_amount_t network_nonvolatile_compressed_under_total;
+ ledger_amount_t network_nonvolatile_compressed_under_max;
} pmap_ledgers_drift;
static void pmap_check_ledgers(pmap_t pmap);
#else /* MACH_ASSERT */
return;
PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
- p, 0, 0, 0, 0);
+ VM_KERNEL_ADDRHIDe(p));
PMAP_LOCK(p);
PMAP_UNLOCK(p);
if (c != 0) {
- PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
- p, 1, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
pmap_assert(p == kernel_pmap);
return; /* still in use */
}
int inuse_ptepages = 0;
zfree(pmap_anchor_zone, p->pm_pml4);
+ zfree(pmap_uanchor_zone, p->pm_upml4);
inuse_ptepages += p->pm_obj_pml4->resident_page_count;
vm_object_deallocate(p->pm_obj_pml4);
ledger_dereference(p->ledger);
zfree(pmap_zone, p);
- PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
}
/*
/*
* Set the physical protection on the
* specified range of this map as requested.
- * Will not increase permissions.
+ *
+ * VERY IMPORTANT: Will *NOT* increase permissions.
+ * pmap_protect_options() should protect the range against any access types
+ * that are not in "prot" but it should never grant extra access.
+ * For example, if "prot" is READ|EXECUTE, that means "remove write
+ * access" but it does *not* mean "add read and execute" access.
+ * VM relies on getting soft-faults to enforce extra checks (code
+ * signing, for example), for example.
+ * New access permissions are granted via pmap_enter() only.
*/
void
pmap_protect_options(
pmap_remove_options(map, sva, eva, options);
return;
}
+
PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
- map,
- (uint32_t) (sva >> 32), (uint32_t) sva,
- (uint32_t) (eva >> 32), (uint32_t) eva);
+ VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(sva),
+ VM_KERNEL_ADDRHIDE(eva));
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled)
set_NX = FALSE;
continue;
if (is_ept) {
- if (prot & VM_PROT_READ)
- pmap_update_pte(spte, 0, PTE_READ(is_ept));
- else
+ if (! (prot & VM_PROT_READ)) {
pmap_update_pte(spte, PTE_READ(is_ept), 0);
+ }
}
- if (prot & VM_PROT_WRITE)
- pmap_update_pte(spte, 0, PTE_WRITE(is_ept));
- else
+ if (! (prot & VM_PROT_WRITE)) {
pmap_update_pte(spte, PTE_WRITE(is_ept), 0);
+ }
+#if DEVELOPMENT || DEBUG
+ else if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) &&
+ map == kernel_pmap) {
+ pmap_update_pte(spte, 0, PTE_WRITE(is_ept));
+ }
+#endif /* DEVELOPMENT || DEBUG */
if (set_NX) {
- if (!is_ept)
+ if (!is_ept) {
pmap_update_pte(spte, 0, INTEL_PTE_NX);
- else
+ } else {
pmap_update_pte(spte, INTEL_EPT_EX, 0);
- } else {
- if (!is_ept)
- pmap_update_pte(spte, INTEL_PTE_NX, 0);
- else
- pmap_update_pte(spte, 0, INTEL_EPT_EX);
+ }
}
num_found++;
}
}
PMAP_UNLOCK(map);
- PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);
}
/* Map a (possibly) autogenned block */
-void
+kern_return_t
pmap_map_block(
- pmap_t pmap,
+ pmap_t pmap,
addr64_t va,
ppnum_t pa,
uint32_t size,
int attr,
__unused unsigned int flags)
{
+ kern_return_t kr;
+ addr64_t original_va = va;
uint32_t page;
int cur_page_size;
if (attr & VM_MEM_SUPERPAGE)
cur_page_size = SUPERPAGE_SIZE;
- else
+ else
cur_page_size = PAGE_SIZE;
for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) {
- pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+ kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+
+ if (kr != KERN_SUCCESS) {
+ /*
+ * This will panic for now, as it is unclear that
+ * removing the mappings is correct.
+ */
+ panic("%s: failed pmap_enter, "
+ "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
+ __FUNCTION__,
+ pmap, va, pa, size, prot, flags);
+
+ pmap_remove(pmap, original_va, va - original_va);
+ return kr;
+ }
+
va += cur_page_size;
pa+=cur_page_size/PAGE_SIZE;
}
+
+ return KERN_SUCCESS;
}
kern_return_t
DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
+ /* With the exception of the kext "basement", the kernel's level 4
+ * pagetables must not be dynamically expanded.
+ */
+ assert(map != kernel_pmap || (vaddr == KERNEL_BASEMENT));
/*
* Allocate a VM page for the pml4 page
*/
| PTE_READ(is_ept)
| (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
| PTE_WRITE(is_ept));
+ pml4_entry_t *upml4p;
+
+ upml4p = pmap64_user_pml4(map, vaddr);
+ pmap_store_pte(upml4p, pa_to_pte(pa)
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
PMAP_UNLOCK(map);
* which is for kexts and is in the 512GB immediately below the kernel..
* XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
*/
- if (map == kernel_pmap &&
- !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))
- panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+ if (__improbable(map == kernel_pmap &&
+ !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))) {
+ if ((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0) {
+ panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+ }
+ }
while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
+ assert((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0);
kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options);
if (pepkr != KERN_SUCCESS)
return pepkr;
VM_PAGE_FREE(m);
- OSAddAtomic(-1, &inuse_ptepages_count);
+ OSAddAtomic(-1, &inuse_ptepages_count);//todo replace all with inlines
PMAP_ZINFO_PFREE(map, PAGE_SIZE);
return KERN_SUCCESS;
}
| PTE_READ(is_ept)
| (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
| PTE_WRITE(is_ept));
+
+ pte = pmap64_user_pml4(pmap, vaddr);
+
+ pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
+ | PTE_READ(is_ept)
+ | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+ | PTE_WRITE(is_ept));
+
}
if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {
{
spl_t s;
+ PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(tpmap));
s = splhigh(); /* Make sure interruptions are disabled */
set_dirbase(tpmap, current_thread(), cpu_number());
splx(s);
+ PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_END);
}
{
unsigned int my_cpu;
unsigned int cpu;
- unsigned int cpu_bit;
+ cpumask_t cpu_bit;
cpumask_t cpus_to_respond = 0;
cpumask_t cpus_to_signal = 0;
cpumask_t cpus_signaled = 0;
cpus_to_signal = pfc->pfc_cpus;
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_START,
- NULL, cpus_to_signal, 0, 0, 0);
+ NULL, cpus_to_signal);
for (cpu = 0, cpu_bit = 1; cpu < real_ncpus && cpus_to_signal; cpu++, cpu_bit <<= 1) {
cpus_to_signal &= ~cpu_bit;
- if (!cpu_datap(cpu)->cpu_running)
+ if (!cpu_is_running(cpu))
continue;
if (pfc->pfc_invalid_global & cpu_bit)
deadline = mach_absolute_time() +
(TLBTimeOut ? TLBTimeOut : LockTimeOut);
boolean_t is_timeout_traced = FALSE;
-
+
/*
* Wait for those other cpus to acknowledge
*/
* as appropriate in the PCID case.
*/
if ((cpus_to_respond & cpu_bit) != 0) {
- if (!cpu_datap(cpu)->cpu_running ||
+ if (!cpu_is_running(cpu) ||
cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
!CPU_CR3_IS_ACTIVE(cpu)) {
cpus_to_respond &= ~cpu_bit;
if (TLBTimeOut == 0) {
if (is_timeout_traced)
continue;
+
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
- NULL, cpus_to_signal, cpus_to_respond, 0, 0);
+ NULL, cpus_to_signal, cpus_to_respond);
+
is_timeout_traced = TRUE;
continue;
}
- pmap_tlb_flush_timeout = TRUE;
orig_acks = NMIPI_acks;
- mp_cpus_NMIPI(cpus_to_respond);
-
- panic("TLB invalidation IPI timeout: "
- "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
- cpus_to_respond, orig_acks, NMIPI_acks);
+ NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+ panic("Uninterruptible processor(s): CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+ cpus_to_respond, orig_acks, NMIPI_acks, deadline);
}
}
}
+
PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_END,
- NULL, cpus_signaled, flush_self, 0, 0);
+ NULL, cpus_signaled, flush_self);
mp_enable_preemption();
}
pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int options, pmap_flush_context *pfc)
{
unsigned int cpu;
- unsigned int cpu_bit;
+ cpumask_t cpu_bit;
cpumask_t cpus_to_signal = 0;
unsigned int my_cpu = cpu_number();
pmap_paddr_t pmap_cr3 = pmap->pm_cr3;
}
PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_START,
- VM_KERNEL_UNSLIDE_OR_PERM(pmap), options, event_startv, event_endv, 0);
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap), options,
+ event_startv, event_endv);
if (is_ept) {
mp_cpus_call(CPUMASK_ALL, ASYNC, invept, (void*)pmap->pm_eptp);
mfence();
}
for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
- if (!cpu_datap(cpu)->cpu_running)
+ if (!cpu_is_running(cpu))
continue;
uint64_t cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
uint64_t cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);
-
+//recall that the shadowed task cr3 is pre-composed
if ((pmap_cr3 == cpu_task_cr3) ||
(pmap_cr3 == cpu_active_cr3) ||
(pmap_is_shared)) {
* as appropriate in the PCID case.
*/
if ((cpus_to_respond & cpu_bit) != 0) {
- if (!cpu_datap(cpu)->cpu_running ||
+ if (!cpu_is_running(cpu) ||
cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
!CPU_CR3_IS_ACTIVE(cpu)) {
cpus_to_respond &= ~cpu_bit;
/* cut tracepoint but don't panic */
if (is_timeout_traced)
continue;
- PMAP_TRACE_CONSTANT(
- PMAP_CODE(PMAP__FLUSH_TLBS_TO),
- VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, cpus_to_respond, 0, 0);
+
+ PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap),
+ cpus_to_signal,
+ cpus_to_respond);
+
is_timeout_traced = TRUE;
continue;
}
- pmap_tlb_flush_timeout = TRUE;
orig_acks = NMIPI_acks;
- mp_cpus_NMIPI(cpus_to_respond);
- panic("TLB invalidation IPI timeout: "
- "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
- cpus_to_respond, orig_acks, NMIPI_acks);
+ NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+ panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+ cpus_to_respond, orig_acks, NMIPI_acks, deadline);
}
}
}
out:
PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_END,
- VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, event_startv, event_endv, 0);
+ VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal,
+ event_startv, event_endv);
}
pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
if (pmap_pcid_ncpus) {
pmap_pcid_validate_current();
- if (cpu_datap(ccpu)->cpu_tlb_invalid_global) {
- cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
- tlb_flush_global();
- }
- else {
- cpu_datap(ccpu)->cpu_tlb_invalid_local = FALSE;
- flush_tlb_raw();
- }
- }
- else {
+ cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
+ tlb_flush_global();
+ } else {
current_cpu_datap()->cpu_tlb_invalid = FALSE;
flush_tlb_raw();
}
void
pmap_update_interrupt(void)
{
- PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START);
if (current_cpu_datap()->cpu_tlb_invalid)
process_pmap_updates();
- PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END,
- 0, 0, 0, 0, 0);
+ PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END);
}
#include <mach/mach_vm.h> /* mach_vm_region_recurse() */
#if MACH_ASSERT
extern int pmap_ledgers_panic;
+extern int pmap_ledgers_panic_leeway;
+
static void
pmap_check_ledgers(
pmap_t pmap)
pmap_ledgers_drift.num_pmaps_checked++;
- ledger_get_balance(pmap->ledger,
- task_ledgers.phys_footprint,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"phys_footprint\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.phys_footprint_over++;
- pmap_ledgers_drift.phys_footprint_over_total += bal;
- if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
- pmap_ledgers_drift.phys_footprint_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.phys_footprint_under++;
- pmap_ledgers_drift.phys_footprint_under_total += bal;
- if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
- pmap_ledgers_drift.phys_footprint_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.internal,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"internal\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.internal_over++;
- pmap_ledgers_drift.internal_over_total += bal;
- if (bal > pmap_ledgers_drift.internal_over_max) {
- pmap_ledgers_drift.internal_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.internal_under++;
- pmap_ledgers_drift.internal_under_total += bal;
- if (bal < pmap_ledgers_drift.internal_under_max) {
- pmap_ledgers_drift.internal_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.internal_compressed,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"internal_compressed\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.internal_compressed_over++;
- pmap_ledgers_drift.internal_compressed_over_total += bal;
- if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
- pmap_ledgers_drift.internal_compressed_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.internal_compressed_under++;
- pmap_ledgers_drift.internal_compressed_under_total += bal;
- if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
- pmap_ledgers_drift.internal_compressed_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.iokit_mapped,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"iokit_mapped\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.iokit_mapped_over++;
- pmap_ledgers_drift.iokit_mapped_over_total += bal;
- if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
- pmap_ledgers_drift.iokit_mapped_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.iokit_mapped_under++;
- pmap_ledgers_drift.iokit_mapped_under_total += bal;
- if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
- pmap_ledgers_drift.iokit_mapped_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.alternate_accounting,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"alternate_accounting\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.alternate_accounting_over++;
- pmap_ledgers_drift.alternate_accounting_over_total += bal;
- if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
- pmap_ledgers_drift.alternate_accounting_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.alternate_accounting_under++;
- pmap_ledgers_drift.alternate_accounting_under_total += bal;
- if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
- pmap_ledgers_drift.alternate_accounting_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.alternate_accounting_compressed,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"alternate_accounting_compressed\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.alternate_accounting_compressed_over++;
- pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
- if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
- pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.alternate_accounting_compressed_under++;
- pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
- if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
- pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.page_table,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"page_table\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.page_table_over++;
- pmap_ledgers_drift.page_table_over_total += bal;
- if (bal > pmap_ledgers_drift.page_table_over_max) {
- pmap_ledgers_drift.page_table_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.page_table_under++;
- pmap_ledgers_drift.page_table_under_total += bal;
- if (bal < pmap_ledgers_drift.page_table_under_max) {
- pmap_ledgers_drift.page_table_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.purgeable_volatile,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"purgeable_volatile\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.purgeable_volatile_over++;
- pmap_ledgers_drift.purgeable_volatile_over_total += bal;
- if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
- pmap_ledgers_drift.purgeable_volatile_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.purgeable_volatile_under++;
- pmap_ledgers_drift.purgeable_volatile_under_total += bal;
- if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
- pmap_ledgers_drift.purgeable_volatile_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.purgeable_nonvolatile,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"purgeable_nonvolatile\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.purgeable_nonvolatile_over++;
- pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
- if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
- pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.purgeable_nonvolatile_under++;
- pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
- if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
- pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.purgeable_volatile_compressed,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"purgeable_volatile_compressed\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.purgeable_volatile_compressed_over++;
- pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
- if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
- pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.purgeable_volatile_compressed_under++;
- pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
- if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
- pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
- }
- }
- }
- ledger_get_balance(pmap->ledger,
- task_ledgers.purgeable_nonvolatile_compressed,
- &bal);
- if (bal != 0) {
- do_panic = TRUE;
- printf("LEDGER BALANCE proc %d (%s) "
- "\"purgeable_nonvolatile_compressed\" = %lld\n",
- pid, procname, bal);
- if (bal > 0) {
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
- if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
- }
- } else {
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
- if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
- pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
- }
- }
- }
+#define LEDGER_CHECK_BALANCE(__LEDGER) \
+MACRO_BEGIN \
+ int panic_on_negative = TRUE; \
+ ledger_get_balance(pmap->ledger, \
+ task_ledgers.__LEDGER, \
+ &bal); \
+ ledger_get_panic_on_negative(pmap->ledger, \
+ task_ledgers.__LEDGER, \
+ &panic_on_negative); \
+ if (bal != 0) { \
+ if (panic_on_negative || \
+ (pmap_ledgers_panic && \
+ pmap_ledgers_panic_leeway > 0 && \
+ (bal > (pmap_ledgers_panic_leeway * PAGE_SIZE) || \
+ bal < (pmap_ledgers_panic_leeway * PAGE_SIZE)))) { \
+ do_panic = TRUE; \
+ } \
+ printf("LEDGER BALANCE proc %d (%s) " \
+ "\"%s\" = %lld\n", \
+ pid, procname, #__LEDGER, bal); \
+ if (bal > 0) { \
+ pmap_ledgers_drift.__LEDGER##_over++; \
+ pmap_ledgers_drift.__LEDGER##_over_total += bal; \
+ if (bal > pmap_ledgers_drift.__LEDGER##_over_max) { \
+ pmap_ledgers_drift.__LEDGER##_over_max = bal; \
+ } \
+ } else if (bal < 0) { \
+ pmap_ledgers_drift.__LEDGER##_under++; \
+ pmap_ledgers_drift.__LEDGER##_under_total += bal; \
+ if (bal < pmap_ledgers_drift.__LEDGER##_under_max) { \
+ pmap_ledgers_drift.__LEDGER##_under_max = bal; \
+ } \
+ } \
+ } \
+MACRO_END
+
+ LEDGER_CHECK_BALANCE(phys_footprint);
+ LEDGER_CHECK_BALANCE(internal);
+ LEDGER_CHECK_BALANCE(internal_compressed);
+ LEDGER_CHECK_BALANCE(iokit_mapped);
+ LEDGER_CHECK_BALANCE(alternate_accounting);
+ LEDGER_CHECK_BALANCE(alternate_accounting_compressed);
+ LEDGER_CHECK_BALANCE(page_table);
+ LEDGER_CHECK_BALANCE(purgeable_volatile);
+ LEDGER_CHECK_BALANCE(purgeable_nonvolatile);
+ LEDGER_CHECK_BALANCE(purgeable_volatile_compressed);
+ LEDGER_CHECK_BALANCE(purgeable_nonvolatile_compressed);
+ LEDGER_CHECK_BALANCE(network_volatile);
+ LEDGER_CHECK_BALANCE(network_nonvolatile);
+ LEDGER_CHECK_BALANCE(network_volatile_compressed);
+ LEDGER_CHECK_BALANCE(network_nonvolatile_compressed);
if (do_panic) {
if (pmap_ledgers_panic) {
}
if (pmap->stats.resident_count != 0 ||
+#if 35156815
+ /*
+ * "wired_count" is unfortunately a bit inaccurate, so let's
+ * tolerate some slight deviation to limit the amount of
+ * somewhat-spurious assertion failures.
+ */
+ pmap->stats.wired_count > 10 ||
+#else /* 35156815 */
pmap->stats.wired_count != 0 ||
+#endif /* 35156815 */
pmap->stats.device != 0 ||
pmap->stats.internal != 0 ||
pmap->stats.external != 0 ||
pmap->stats.reusable != 0 ||
pmap->stats.compressed != 0) {
- if (pmap_stats_assert) {
+ if (pmap_stats_assert &&
+ pmap->pmap_stats_assert) {
panic("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
pmap, pid, procname,
pmap->stats.resident_count,
pmap->pmap_pid = pid;
strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
+ if (pmap_ledgers_panic_leeway) {
+ /*
+ * XXX FBDP
+ * Some processes somehow trigger some issues that make
+ * the pmap stats and ledgers go off track, causing
+ * some assertion failures and ledger panics.
+ * Turn off the sanity checks if we allow some ledger leeway
+ * because of that. We'll still do a final check in
+ * pmap_check_ledgers() for discrepancies larger than the
+ * allowed leeway after the address space has been fully
+ * cleaned up.
+ */
+ pmap->pmap_stats_assert = FALSE;
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.phys_footprint);
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.internal);
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.internal_compressed);
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.iokit_mapped);
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.alternate_accounting);
+ ledger_disable_panic_on_negative(pmap->ledger,
+ task_ledgers.alternate_accounting_compressed);
+ }
}
#endif /* MACH_ASSERT */
return;
panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
}
+
projects / apple / xnu.git / blobdiff