X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..527f99514973766e9c0382a4d8550dfb00f54939:/osfmk/i386/pmap.h diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h index 939e47174..3458ce7cf 100644 --- a/osfmk/i386/pmap.h +++ b/osfmk/i386/pmap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,6 @@ #ifndef ASSEMBLER - #include #include #include @@ -96,9 +95,6 @@ #define intel_ptob(x) i386_ptob(x) #define intel_round_page(x) i386_round_page(x) #define intel_trunc_page(x) i386_trunc_page(x) -#define trunc_intel_to_vm(x) trunc_i386_to_vm(x) -#define round_intel_to_vm(x) round_i386_to_vm(x) -#define vm_to_intel(x) vm_to_i386(x) /* * i386/i486/i860 Page Table Entry @@ -113,10 +109,7 @@ #define PTESHIFT 12ULL - -#ifdef __x86_64__ #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL) -#endif #define PDESIZE sizeof(pd_entry_t) /* for assembly files */ #define PTESIZE sizeof(pt_entry_t) /* for assembly files */ @@ -170,17 +163,25 @@ typedef uint64_t pt_entry_t; typedef uint64_t pmap_paddr_t; -#if DEBUG +#if DEVELOPMENT || DEBUG #define PMAP_ASSERT 1 +extern int pmap_asserts_enabled; +extern int pmap_asserts_traced; #endif + #if PMAP_ASSERT -#define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) +#define pmap_assert(ex) (pmap_asserts_enabled ? ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) : (void)0) #define pmap_assert2(ex, fmt, args...) \ do { \ - if (!(ex)) { \ - kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ - panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ + if (__improbable(pmap_asserts_enabled && !(ex))) { \ + if (pmap_asserts_traced) { \ + KERNEL_DEBUG_CONSTANT(0xDEAD1000, __builtin_return_address(0), __LINE__, 0, 0, 0); \ + kdebug_enable = 0; \ + } else { \ + kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ + panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ + } \ } \ } while(0) #else @@ -189,11 +190,7 @@ typedef uint64_t pmap_paddr_t; #endif /* superpages */ -#ifdef __x86_64__ #define SUPERPAGE_NBASEPAGES 512 -#else -#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */ -#endif /* * Atomic 64-bit store of a page table entry. @@ -215,8 +212,11 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t)))) #define KERNEL_PML4_INDEX 511 -#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ +#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */ +#define KERNEL_KASAN_PML4_INDEX0 508 +#define KERNEL_KASAN_PML4_INDEX1 507 +#define KERNEL_DBLMAP_PML4_INDEX (506) #define KERNEL_BASE (0ULL - NBPML4) #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4) @@ -226,6 +226,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) /* ?? intel ?? */ #define VM_WIMG_IO (VM_MEM_COHERENT | \ VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED) +#define VM_WIMG_POSTED VM_WIMG_IO #define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED) /* write combining mode, aka store gather */ #define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) @@ -307,7 +308,17 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) /* This is conservative, but suffices */ #define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54)) -#define INTEL_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/ +#define INTEL_PTE_COMPRESSED (1ULL << 62) /* marker, for invalid PTE only -- ignored by hardware for both regular/EPT entries*/ +#define INTEL_PTE_COMPRESSED_ALT (1ULL << 61) /* compressed but with "alternate accounting" */ + +#define INTEL_PTE_COMPRESSED_MASK (INTEL_PTE_COMPRESSED | \ + INTEL_PTE_COMPRESSED_ALT) +#define PTE_IS_COMPRESSED(x) \ + ((((x) & INTEL_PTE_VALID) == 0) && /* PTE is not valid... */ \ + ((x) & INTEL_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ + ((!((x) & ~INTEL_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ + (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \ + &(x), (x), (x) & ~INTEL_PTE_COMPRESSED_MASK), FALSE))) #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */ #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */ @@ -407,7 +418,8 @@ extern boolean_t pmap_ept_support_ad; #define PTE_READ(is_ept) ((is_ept) ? INTEL_EPT_READ : INTEL_PTE_VALID) #define PTE_WRITE(is_ept) ((is_ept) ? INTEL_EPT_WRITE : INTEL_PTE_WRITE) #define PTE_PS INTEL_PTE_PS -#define PTE_COMPRESSED INTEL_COMPRESSED +#define PTE_COMPRESSED INTEL_PTE_COMPRESSED +#define PTE_COMPRESSED_ALT INTEL_PTE_COMPRESSED_ALT #define PTE_NCACHE(is_ept) ((is_ept) ? INTEL_EPT_NCACHE : INTEL_PTE_NCACHE) #define PTE_WTHRU(is_ept) ((is_ept) ? INTEL_EPT_WTHRU : INTEL_PTE_WTHRU) #define PTE_REF(is_ept) ((is_ept) ? INTEL_EPT_REF : INTEL_PTE_REF) @@ -434,7 +446,6 @@ extern pt_entry_t *PTmap; extern pdpt_entry_t *IdlePDPT; extern pml4_entry_t *IdlePML4; extern boolean_t no_shared_cr3; -extern addr64_t kernel64_cr3; extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */ extern uint64_t pmap_pv_hashlist_walks; @@ -442,9 +453,6 @@ extern uint64_t pmap_pv_hashlist_cnts; extern uint32_t pmap_pv_hashlist_max; extern uint32_t pmap_kernel_text_ps; - - -#ifdef __x86_64__ #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK)) extern uint64_t physmap_base, physmap_max; @@ -466,6 +474,30 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { } #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x))) +#if MACH_KERNEL_PRIVATE +extern uint64_t dblmap_base, dblmap_max, dblmap_dist; + +static inline uint64_t DBLMAP_CHECK(uintptr_t x) { + uint64_t dbladdr = (uint64_t)x + dblmap_dist; + if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) { + panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx", + (uint64_t)x, dbladdr, dblmap_base, dblmap_max); + } + return dbladdr; + +} +#define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x)) +extern uint64_t ldt_alias_offset; +static inline uint64_t LDTALIAS_CHECK(uintptr_t x) { + uint64_t dbladdr = (uint64_t)x + ldt_alias_offset; + if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) { + panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx", + (uint64_t)x, dbladdr, dblmap_base, dblmap_max); + } + return dbladdr; +} +#define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x)) +#endif /* * For KASLR, we alias the master processor's IDT and GDT at fixed @@ -473,12 +505,13 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS) * The low global vector page is mapped at a fixed alias also. */ -#define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000) -#define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000) #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000) -#define CPU_GDT_ALIAS(_cpu) (LOWGLOBAL_ALIAS + (0x1000*(_cpu))) -#endif /*__x86_64__ */ +/* + * This indicates (roughly) where there is free space for the VM + * to use for the heap; this does not need to be precise. + */ +#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS #include @@ -490,23 +523,28 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { struct pmap { decl_simple_lock_data(,lock) /* lock on map */ - pmap_paddr_t pm_cr3; /* physical addr */ - pmap_paddr_t pm_eptp; /* EPTP */ - boolean_t pm_shared; - pd_entry_t *dirbase; /* page directory pointer */ - vm_object_t pm_obj; /* object to hold pde's */ + pmap_paddr_t pm_cr3; /* Kernel+user shared PML4 physical*/ + pmap_paddr_t pm_ucr3; /* Mirrored user PML4 physical */ task_map_t pm_task_map; - pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */ - pml4_entry_t *pm_pml4; /* VKA of top level */ - vm_object_t pm_obj_pdpt; /* holds pdpt pages */ - vm_object_t pm_obj_pml4; /* holds pml4 pages */ + boolean_t pm_shared; + boolean_t pagezero_accessible; #define PMAP_PCID_MAX_CPUS MAX_CPUS /* Must be a multiple of 8 */ pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS]; volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS]; struct pmap_statistics stats; /* map statistics */ int ref_count; /* reference count */ int nx_enabled; + pml4_entry_t *pm_pml4; /* VKA of top level */ + pml4_entry_t *pm_upml4; /* Shadow VKA of top level */ + vm_object_t pm_obj; /* object to hold pde's */ + vm_object_t pm_obj_pdpt; /* holds pdpt pages */ + vm_object_t pm_obj_pml4; /* holds pml4 pages */ + pmap_paddr_t pm_eptp; /* EPTP */ ledger_t ledger; /* ledger tracking phys mappings */ +#if MACH_ASSERT + int pmap_pid; + char pmap_procname[17]; +#endif /* MACH_ASSERT */ }; static inline boolean_t @@ -571,22 +609,48 @@ extern pmap_memory_region_t pmap_memory_regions[]; #include static inline void -set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) { +set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) { int ccpu = my_cpu; - cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3; + uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3; + cpu_datap(ccpu)->cpu_task_cr3 = pcr3; + cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3; + + cpu_datap(ccpu)->cpu_ucr3 = ucr3; + cpu_shadowp(ccpu)->cpu_ucr3 = ucr3; + cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map; + + assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE)); + assert(ccpu == cpu_number()); /* * Switch cr3 if necessary * - unless running with no_shared_cr3 debugging mode * and we're not on the kernel's cr3 (after pre-empted copyio) */ + boolean_t nopagezero = tpmap->pagezero_accessible; + boolean_t priorpagezero = cpu_datap(ccpu)->cpu_pagezero_mapped; + cpu_datap(ccpu)->cpu_pagezero_mapped = nopagezero; + if (__probable(!no_shared_cr3)) { - if (get_cr3_base() != tpmap->pm_cr3) { + if (__improbable(nopagezero)) { + boolean_t copyio_active = ((thread->machine.specFlags & CopyIOActive) != 0); if (pmap_pcid_ncpus) { - pmap_pcid_activate(tpmap, ccpu); + pmap_pcid_activate(tpmap, ccpu, TRUE, copyio_active); + } else { + if (copyio_active) { + if (get_cr3_base() != tpmap->pm_cr3) { + set_cr3_raw(tpmap->pm_cr3); + } + } else if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) { + set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3); + } } - else + } else if ((get_cr3_base() != tpmap->pm_cr3) || priorpagezero) { + if (pmap_pcid_ncpus) { + pmap_pcid_activate(tpmap, ccpu, FALSE, FALSE); + } else { set_cr3_raw(tpmap->pm_cr3); + } } } else { if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) @@ -601,10 +665,6 @@ set_dirbase(pmap_t tpmap, __unused thread_t thread, int my_cpu) { extern void process_pmap_updates(void); extern void pmap_update_interrupt(void); -/* - * Machine dependent routines that are used only for i386/i486/i860. - */ - extern addr64_t (kvtophys)( vm_offset_t addr); @@ -612,23 +672,6 @@ extern kern_return_t pmap_expand( pmap_t pmap, vm_map_offset_t addr, unsigned int options); -#if !defined(__x86_64__) -extern pt_entry_t *pmap_pte( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pd_entry_t *pmap_pde( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pd_entry_t *pmap64_pde( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pdpt_entry_t *pmap64_pdpt( - struct pmap *pmap, - vm_map_offset_t addr); -#endif extern vm_offset_t pmap_map( vm_offset_t virt, vm_map_offset_t start, @@ -642,7 +685,6 @@ extern vm_offset_t pmap_map_bd( vm_map_offset_t end, vm_prot_t prot, unsigned int flags); - extern void pmap_bootstrap( vm_offset_t load_start, boolean_t IA32e); @@ -666,15 +708,15 @@ extern void pmap_cpu_free( struct cpu_pmap *cp); #endif -extern void pmap_map_block( - pmap_t pmap, +extern kern_return_t pmap_map_block( + pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags); - + extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); @@ -683,7 +725,7 @@ extern void pmap_cpu_init(void); extern void pmap_disable_NX(pmap_t pmap); extern void pt_fake_zone_init(int); -extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, +extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, uint64_t *, int *, int *, int *); extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2)); @@ -694,9 +736,9 @@ extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__pr #include - + #define PMAP_ACTIVATE_MAP(map, thread, my_cpu) { \ - register pmap_t tpmap; \ + pmap_t tpmap; \ \ tpmap = vm_map_pmap(map); \ set_dirbase(tpmap, thread, my_cpu); \ @@ -704,20 +746,11 @@ extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__pr #if defined(__x86_64__) #define PMAP_DEACTIVATE_MAP(map, thread, ccpu) \ - pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE); + pmap_assert2((pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu) == (get_cr3_raw() & 0xFFF)) : TRUE),"PCIDs: 0x%x, active PCID: 0x%x, CR3: 0x%lx, pmap_cr3: 0x%llx, kernel_cr3: 0x%llx, kernel pmap cr3: 0x%llx, CPU active PCID: 0x%x, CPU kernel PCID: 0x%x, specflags: 0x%x, pagezero: 0x%x", pmap_pcid_ncpus, pcid_for_pmap_cpu_tuple(map->pmap, thread, ccpu), get_cr3_raw(), map->pmap->pm_cr3, cpu_datap(ccpu)->cpu_kernel_cr3, kernel_pmap->pm_cr3, cpu_datap(ccpu)->cpu_active_pcid, cpu_datap(ccpu)->cpu_kernel_pcid, thread->machine.specFlags, map->pmap->pagezero_accessible); #else #define PMAP_DEACTIVATE_MAP(map, thread) #endif -#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \ - \ - pmap_assert(ml_get_interrupts_enabled() == FALSE); \ - if (old_th->map != new_th->map) { \ - PMAP_DEACTIVATE_MAP(old_th->map, old_th, my_cpu); \ - PMAP_ACTIVATE_MAP(new_th->map, new_th, my_cpu); \ - } \ -} - #if NCOPY_WINDOWS > 0 #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ spl_t spl; \ @@ -821,10 +854,15 @@ extern boolean_t pmap_is_empty(pmap_t pmap, kern_return_t pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t); +#if MACH_ASSERT +extern int pmap_stats_assert; +#define PMAP_STATS_ASSERTF(args) \ + MACRO_BEGIN \ + if (pmap_stats_assert) assertf args; \ + MACRO_END +#else /* MACH_ASSERT */ +#define PMAP_STATS_ASSERTF(args) +#endif /* MACH_ASSERT */ #endif /* ASSEMBLER */ - - #endif /* _PMAP_MACHINE_ */ - - #endif /* KERNEL_PRIVATE */