/*
- * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#ifndef ASSEMBLER
-
#include <mach/kern_return.h>
#include <mach/machine/vm_types.h>
#include <mach/vm_prot.h>
#define intel_ptob(x) i386_ptob(x)
#define intel_round_page(x) i386_round_page(x)
#define intel_trunc_page(x) i386_trunc_page(x)
-#define trunc_intel_to_vm(x) trunc_i386_to_vm(x)
-#define round_intel_to_vm(x) round_i386_to_vm(x)
-#define vm_to_intel(x) vm_to_i386(x)
/*
* i386/i486/i860 Page Table Entry
#define PTESHIFT 12ULL
-
-#ifdef __x86_64__
#define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL)
-#endif
#define PDESIZE sizeof(pd_entry_t) /* for assembly files */
#define PTESIZE sizeof(pt_entry_t) /* for assembly files */
typedef uint64_t pmap_paddr_t;
-#if DEBUG
+#if DEVELOPMENT || DEBUG
#define PMAP_ASSERT 1
+extern int pmap_asserts_enabled;
+extern int pmap_asserts_traced;
#endif
+
#if PMAP_ASSERT
-#define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+#define pmap_assert(ex) (pmap_asserts_enabled ? ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) : (void)0)
#define pmap_assert2(ex, fmt, args...) \
do { \
- if (!(ex)) { \
- kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
- panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
+ if (__improbable(pmap_asserts_enabled && !(ex))) { \
+ if (pmap_asserts_traced) { \
+ KERNEL_DEBUG_CONSTANT(0xDEAD1000, __builtin_return_address(0), __LINE__, 0, 0, 0); \
+ kdebug_enable = 0; \
+ } else { \
+ kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
+ panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \
+ } \
} \
} while(0)
#else
#endif
/* superpages */
-#ifdef __x86_64__
#define SUPERPAGE_NBASEPAGES 512
-#else
-#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
-#endif
/*
* Atomic 64-bit store of a page table entry.
#define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
#define KERNEL_PML4_INDEX 511
-#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
+#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
#define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */
+#define KERNEL_KASAN_PML4_INDEX0 508
+#define KERNEL_KASAN_PML4_INDEX1 507
+#define KERNEL_DBLMAP_PML4_INDEX (506)
#define KERNEL_BASE (0ULL - NBPML4)
#define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
/* ?? intel ?? */
#define VM_WIMG_IO (VM_MEM_COHERENT | \
VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
+#define VM_WIMG_POSTED VM_WIMG_IO
#define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
/* write combining mode, aka store gather */
#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
/* This is conservative, but suffices */
#define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
-#define INTEL_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
+#define INTEL_PTE_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/
+#define INTEL_PTE_COMPRESSED_ALT (1ULL << 61) /* compressed but with "alternate accounting" */
+
+#define INTEL_PTE_COMPRESSED_MASK (INTEL_PTE_COMPRESSED | \
+ INTEL_PTE_COMPRESSED_ALT)
+#define PTE_IS_COMPRESSED(x) \
+ ((((x) & INTEL_PTE_VALID) == 0) && /* PTE is not valid... */ \
+ ((x) & INTEL_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
+ ((!((x) & ~INTEL_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
+ (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
+ &(x), (x), (x) & ~INTEL_PTE_COMPRESSED_MASK), FALSE)))
#define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
#define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
#define PTE_READ(is_ept) ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID)
#define PTE_WRITE(is_ept) ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE)
#define PTE_PS INTEL_PTE_PS
-#define PTE_COMPRESSED INTEL_COMPRESSED
+#define PTE_COMPRESSED INTEL_PTE_COMPRESSED
+#define PTE_COMPRESSED_ALT INTEL_PTE_COMPRESSED_ALT
#define PTE_NCACHE(is_ept) ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE)
#define PTE_WTHRU(is_ept) ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU)
#define PTE_REF(is_ept) ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF)
extern pdpt_entry_t *IdlePDPT;
extern pml4_entry_t *IdlePML4;
extern boolean_t no_shared_cr3;
-extern addr64_t kernel64_cr3;
extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */
extern uint64_t pmap_pv_hashlist_walks;
extern uint32_t pmap_pv_hashlist_max;
extern uint32_t pmap_kernel_text_ps;
-
-
-#ifdef __x86_64__
#define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
extern uint64_t physmap_base, physmap_max;
}
#define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
+#if MACH_KERNEL_PRIVATE
+extern uint64_t dblmap_base, dblmap_max, dblmap_dist;
+
+static inline uint64_t DBLMAP_CHECK(uintptr_t x) {
+ uint64_t dbladdr = (uint64_t)x + dblmap_dist;
+ if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+ panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+ (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+ }
+ return dbladdr;
+
+}
+#define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x))
+extern uint64_t ldt_alias_offset;
+static inline uint64_t LDTALIAS_CHECK(uintptr_t x) {
+ uint64_t dbladdr = (uint64_t)x + ldt_alias_offset;
+ if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+ panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+ (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+ }
+ return dbladdr;
+}
+#define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x))
+#endif
/*
* For KASLR, we alias the master processor's IDT and GDT at fixed
* And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
* The low global vector page is mapped at a fixed alias also.
*/
-#define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000)
-#define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000)
#define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000)
-#define CPU_GDT_ALIAS(_cpu) (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))
-#endif /*__x86_64__ */
+/*
+ * This indicates (roughly) where there is free space for the VM
+ * to use for the heap; this does not need to be precise.
+ */
+#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
#include <vm/vm_page.h>
struct pmap {
decl_simple_lock_data(,lock) /* lock on map */
- pmap_paddr_t pm_cr3; /* physical addr */
- pmap_paddr_t pm_eptp; /* EPTP */
- boolean_t pm_shared;
- pd_entry_t *dirbase; /* page directory pointer */
- vm_object_t pm_obj; /* object to hold pde's */
+ pmap_paddr_t pm_cr3; /* Kernel+user shared PML4 physical*/
+ pmap_paddr_t pm_ucr3; /* Mirrored user PML4 physical */
task_map_t pm_task_map;
- pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
- pml4_entry_t *pm_pml4; /* VKA of top level */
- vm_object_t pm_obj_pdpt; /* holds pdpt pages */
- vm_object_t pm_obj_pml4; /* holds pml4 pages */
+ boolean_t pm_shared;
+ boolean_t pagezero_accessible;
#define PMAP_PCID_MAX_CPUS MAX_CPUS /* Must be a multiple of 8 */
pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
struct pmap_statistics stats; /* map statistics */
int ref_count; /* reference count */
int nx_enabled;
+ pml4_entry_t *pm_pml4; /* VKA of top level */
+ pml4_entry_t *pm_upml4; /* Shadow VKA of top level */
+ vm_object_t pm_obj; /* object to hold pde's */
+ vm_object_t pm_obj_pdpt; /* holds pdpt pages */
+ vm_object_t pm_obj_pml4; /* holds pml4 pages */
+ pmap_paddr_t pm_eptp; /* EPTP */
ledger_t ledger; /* ledger tracking phys mappings */
+#if MACH_ASSERT
+ int pmap_pid;
+ char pmap_procname[17];
+#endif /* MACH_ASSERT */
};
static inline boolean_t
#include <i386/pmap_pcid.h>
static inline void
-set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) {
+set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
int ccpu = my_cpu;
- cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
+ uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3;
+ cpu_datap(ccpu)->cpu_task_cr3 = pcr3;
+ cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3;
+
+ cpu_datap(ccpu)->cpu_ucr3 = ucr3;
+ cpu_shadowp(ccpu)->cpu_ucr3 = ucr3;
+
cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
+
+ assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));
+ assert(ccpu == cpu_number());
/*
* Switch cr3 if necessary
* - unless running with no_shared_cr3 debugging mode
* and we're not on the kernel's cr3 (after pre-empted copyio)
*/
+ boolean_t nopagezero = tpmap->pagezero_accessible;
+ boolean_t priorpagezero = cpu_datap(ccpu)->cpu_pagezero_mapped;
+ cpu_datap(ccpu)->cpu_pagezero_mapped = nopagezero;
+
if (__probable(!no_shared_cr3)) {
- if (get_cr3_base() != tpmap->pm_cr3) {
+ if (__improbable(nopagezero)) {
+ boolean_t copyio_active = ((thread->machine.specFlags & CopyIOActive) != 0);
if (pmap_pcid_ncpus) {
- pmap_pcid_activate(tpmap, ccpu);
+ pmap_pcid_activate(tpmap, ccpu, TRUE, copyio_active);
+ } else {
+ if (copyio_active) {
+ if (get_cr3_base() != tpmap->pm_cr3) {
+ set_cr3_raw(tpmap->pm_cr3);
+ }
+ } else if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) {
+ set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
+ }
}
- else
+ } else if ((get_cr3_base() != tpmap->pm_cr3) || priorpagezero) {
+ if (pmap_pcid_ncpus) {
+ pmap_pcid_activate(tpmap, ccpu, FALSE, FALSE);
+ } else {
set_cr3_raw(tpmap->pm_cr3);
+ }
}
} else {
if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
extern void process_pmap_updates(void);
extern void pmap_update_interrupt(void);
-/*
- * Machine dependent routines that are used only for i386/i486/i860.
- */
-
extern addr64_t (kvtophys)(
vm_offset_t addr);
pmap_t pmap,
vm_map_offset_t addr,
unsigned int options);
-#if !defined(__x86_64__)
-extern pt_entry_t *pmap_pte(
- struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pd_entry_t *pmap_pde(
- struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pd_entry_t *pmap64_pde(
- struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pdpt_entry_t *pmap64_pdpt(
- struct pmap *pmap,
- vm_map_offset_t addr);
-#endif
extern vm_offset_t pmap_map(
vm_offset_t virt,
vm_map_offset_t start,
vm_map_offset_t end,
vm_prot_t prot,
unsigned int flags);
-
extern void pmap_bootstrap(
vm_offset_t load_start,
boolean_t IA32e);
struct cpu_pmap *cp);
#endif
-extern void pmap_map_block(
- pmap_t pmap,
+extern kern_return_t pmap_map_block(
+ pmap_t pmap,
addr64_t va,
ppnum_t pa,
uint32_t size,
vm_prot_t prot,
int attr,
unsigned int flags);
-
+
extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
extern void pmap_disable_NX(pmap_t pmap);
extern void pt_fake_zone_init(int);
-extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
+extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
uint64_t *, int *, int *, int *);
extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
#include <kern/spl.h>
-
+
#define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \
- register pmap_t tpmap; \
+ pmap_t tpmap; \
\
tpmap = vm_map_pmap(map); \
set_dirbase(tpmap, thread, my_cpu); \
#if defined(__x86_64__)
#define PMAP_DEACTIVATE_MAP(map, thread, ccpu) \
- pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE);
+ pmap_assert2((pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE),"PCIDs: 0x%x, active PCID: 0x%x, CR3: 0x%lx, pmap_cr3: 0x%llx, kernel_cr3: 0x%llx, kernel pmap cr3: 0x%llx, CPU active PCID: 0x%x, CPU kernel PCID: 0x%x, specflags: 0x%x, pagezero: 0x%x", pmap_pcid_ncpus, pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu), get_cr3_raw(), map->pmap->pm_cr3, cpu_datap(ccpu)->cpu_kernel_cr3, kernel_pmap->pm_cr3, cpu_datap(ccpu)->cpu_active_pcid, cpu_datap(ccpu)->cpu_kernel_pcid, thread->machine.specFlags, map->pmap->pagezero_accessible);
#else
#define PMAP_DEACTIVATE_MAP(map, thread)
#endif
-#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
- \
- pmap_assert(ml_get_interrupts_enabled() == FALSE); \
- if (old_th->map != new_th->map) { \
- PMAP_DEACTIVATE_MAP(old_th->map, old_th, my_cpu); \
- PMAP_ACTIVATE_MAP(new_th->map, new_th, my_cpu); \
- } \
-}
-
#if NCOPY_WINDOWS > 0
#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
spl_t spl; \
kern_return_t
pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
+#if MACH_ASSERT
+extern int pmap_stats_assert;
+#define PMAP_STATS_ASSERTF(args) \
+ MACRO_BEGIN \
+ if (pmap_stats_assert) assertf args; \
+ MACRO_END
+#else /* MACH_ASSERT */
+#define PMAP_STATS_ASSERTF(args)
+#endif /* MACH_ASSERT */
#endif /* ASSEMBLER */
-
-
#endif /* _PMAP_MACHINE_ */
-
-
#endif /* KERNEL_PRIVATE */
projects / apple / xnu.git / blobdiff