X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5d5c5d0d5b79ade9a973d55186ffda2638ba2b6e..6d2010ae8f7a6078e10b361c6962983bab233e0f:/osfmk/i386/pmap.h diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h index f354581c3..44b6bf742 100644 --- a/osfmk/i386/pmap.h +++ b/osfmk/i386/pmap.h @@ -1,32 +1,29 @@ - /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -83,10 +80,13 @@ #include #include #include +#include #include #include +#include + /* * Define the generic in terms of the specific */ @@ -114,10 +114,20 @@ #define PTESHIFT 12 + +#define INITPT_SEG_BASE 0x100000 +#define INITGDT_SEG_BASE 0x106000 +#define SLEEP_SEG_BASE 0x107000 + +#ifdef __x86_64__ +#define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL) +#endif + #define PDESIZE sizeof(pd_entry_t) /* for assembly files */ #define PTESIZE sizeof(pt_entry_t) /* for assembly files */ #define INTEL_OFFMASK (I386_PGBYTES - 1) +#define INTEL_LOFFMASK (I386_LPGBYTES - 1) #define PG_FRAME 0x000FFFFFFFFFF000ULL #define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t))) #define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t))) @@ -128,6 +138,8 @@ #define NBPDE (1 << PDESHIFT) #define PDEMASK (NBPDE - 1) +#define PTE_PER_PAGE 512 /* number of PTE's per page on any level */ + /* cleanly define parameters for all the page table levels */ typedef uint64_t pml4_entry_t; #define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t))) @@ -163,12 +175,38 @@ typedef uint64_t pt_entry_t; typedef uint64_t pmap_paddr_t; +#if DEBUG +#define PMAP_ASSERT 1 +#endif +#if PMAP_ASSERT +#define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) + +#define pmap_assert2(ex, fmt, args...) \ + do { \ + if (!(ex)) { \ + kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ + panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0), ##args); \ + } \ + } while(0) +#else +#define pmap_assert(ex) +#define pmap_assert2(ex, fmt, args...) +#endif + +/* superpages */ +#ifdef __x86_64__ +#define SUPERPAGE_NBASEPAGES 512 +#else +#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */ +#endif + /* * Atomic 64-bit store of a page table entry. */ static inline void pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) { +#ifdef __i386__ /* * Load the new value into %ecx:%ebx * Load the old value into %edx:%eax @@ -176,7 +214,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) * If the compare succeeds, the new value will have been stored. * Otherwise, the old value changed and reloaded, so try again. */ - asm volatile( + __asm__ volatile( " movl (%0), %%eax \n\t" " movl 4(%0), %%edx \n\t" "1: \n\t" @@ -187,14 +225,74 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) "b" ((uint32_t)value), "c" ((uint32_t)(value >> 32)) : "eax", "edx", "memory"); +#else + /* + * In the 32-bit kernel a compare-and-exchange loop was + * required to provide atomicity. For K64, life is easier: + */ + *entryp = value; +#endif +} + +/* + * Atomic 64-bit compare and exchange of a page table entry. + */ +static inline boolean_t +pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new) +{ + boolean_t ret; + +#ifdef __i386__ + /* + * Load the old value into %edx:%eax + * Load the new value into %ecx:%ebx + * Compare-exchange-8bytes at address entryp (loaded in %edi) + * If the compare succeeds, the new value is stored, return TRUE. + * Otherwise, no swap is made, return FALSE. + */ + asm volatile( + " lock; cmpxchg8b (%1) \n\t" + " setz %%al \n\t" + " movzbl %%al,%0" + : "=a" (ret) + : "D" (entryp), + "a" ((uint32_t)old), + "d" ((uint32_t)(old >> 32)), + "b" ((uint32_t)new), + "c" ((uint32_t)(new >> 32)) + : "memory"); +#else + /* + * Load the old value into %rax + * Load the new value into another register + * Compare-exchange-quad at address entryp + * If the compare succeeds, the new value is stored, return TRUE. + * Otherwise, no swap is made, return FALSE. + */ + asm volatile( + " lock; cmpxchgq %2,(%3) \n\t" + " setz %%al \n\t" + " movzbl %%al,%0" + : "=a" (ret) + : "a" (old), + "r" (new), + "r" (entryp) + : "memory"); +#endif + return ret; } +#define pmap_update_pte(entryp, old, new) \ + while (!pmap_cmpx_pte((entryp), (old), (new))) + + /* in 64 bit spaces, the number of each type of page in the page tables */ #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t)))) #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t)))) #define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t)))) #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t)))) +#ifdef __i386__ /* * The 64-bit kernel is remapped in uber-space which is at the base * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is, @@ -203,6 +301,13 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) #define KERNEL_UBER_PML4_INDEX 511 #define KERNEL_UBER_BASE (0ULL - NBPML4) #define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32)) +#else +#define KERNEL_PML4_INDEX 511 +#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ +#define KERNEL_PHYSMAP_INDEX 509 /* virtual to physical map */ +#define KERNEL_BASE (0ULL - NBPML4) +#define KERNEL_BASEMENT (KERNEL_BASE - NBPML4) +#endif #define VM_WIMG_COPYBACK VM_MEM_COHERENT #define VM_WIMG_DEFAULT VM_MEM_COHERENT @@ -216,8 +321,18 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) /* * Pte related macros */ +#ifdef __i386__ #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<> PDESHIFT) & PDEMASK) +#define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK) +#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK) +#define PMAP_INVALID_PDPTNUM (~0ULL) #define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1)) #define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1)) #define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1)) + /* * Convert page descriptor index to user virtual address */ @@ -314,15 +434,18 @@ enum high_fixed_addresses { #define INTEL_PTE_NCACHE 0x00000010 #define INTEL_PTE_REF 0x00000020 #define INTEL_PTE_MOD 0x00000040 -#define INTEL_PTE_PS 0x00000080 -#define INTEL_PTE_GLOBAL 0x00000100 +#define INTEL_PTE_PS 0x00000080 +#define INTEL_PTE_PTA 0x00000080 +#define INTEL_PTE_GLOBAL 0x00000100 #define INTEL_PTE_WIRED 0x00000200 +#define INTEL_PDPTE_NESTED 0x00000400 #define INTEL_PTE_PFN PG_FRAME -#define INTEL_PTE_PTA 0x00000080 #define INTEL_PTE_NX (1ULL << 63) #define INTEL_PTE_INVALID 0 +/* This is conservative, but suffices */ +#define INTEL_PTE_RSVD ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54)) #define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */ #define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */ @@ -349,27 +472,43 @@ enum high_fixed_addresses { * and directories. */ -extern pt_entry_t PTmap[], APTmap[], Upte; -extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde; - -extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */ -extern pdpt_entry_t *IdlePDPT; - -extern pmap_paddr_t lo_kernel_cr3; - -extern pml4_entry_t *IdlePML4; -extern pdpt_entry_t *IdlePDPT64; -extern addr64_t kernel64_cr3; -extern boolean_t no_shared_cr3; - +#ifdef __i386__ +extern pt_entry_t PTmap[], APTmap[], Upte; +extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde; +extern pmap_paddr_t lo_kernel_cr3; +extern pdpt_entry_t *IdlePDPT64; +#else +extern pt_entry_t *PTmap; +#endif +extern boolean_t no_shared_cr3; +extern addr64_t kernel64_cr3; +extern pd_entry_t *IdlePTD; /* physical addr of "Idle" state PTD */ +extern pdpt_entry_t IdlePDPT[]; +extern pml4_entry_t IdlePML4[]; + +extern uint64_t pmap_pv_hashlist_walks; +extern uint64_t pmap_pv_hashlist_cnts; +extern uint32_t pmap_pv_hashlist_max; +extern uint32_t pmap_kernel_text_ps; + +#ifdef __i386__ /* + * ** i386 ** * virtual address to page table entry and * to physical address. Likewise for alternate address space. * Note: these work recursively, thus vtopte of a pte will give * the corresponding pde that in turn maps it. */ + #define vtopte(va) (PTmap + i386_btop((vm_offset_t)va)) +#endif +#ifdef __x86_64__ +#define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK)) + +#define PHYSMAP_BASE KVADDR(KERNEL_PHYSMAP_INDEX,0,0,0) +#define PHYSMAP_PTOV(x) ((void *)(((uint64_t)(x)) + PHYSMAP_BASE)) +#endif typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */ /* changed by other processors */ @@ -387,25 +526,30 @@ struct md_page { */ struct pmap { + decl_simple_lock_data(,lock) /* lock on map */ + pmap_paddr_t pm_cr3; /* physical addr */ + boolean_t pm_shared; pd_entry_t *dirbase; /* page directory pointer */ +#ifdef __i386__ pmap_paddr_t pdirbase; /* phys. address of dirbase */ - vm_object_t pm_obj; /* object to hold pde's */ - int ref_count; /* reference count */ - int nx_enabled; - boolean_t pm_64bit; - boolean_t pm_kernel_cr3; - decl_simple_lock_data(,lock) /* lock on map */ - struct pmap_statistics stats; /* map statistics */ vm_offset_t pm_hold; /* true pdpt zalloc addr */ - pmap_paddr_t pm_cr3; /* physical addr */ +#endif + vm_object_t pm_obj; /* object to hold pde's */ + task_map_t pm_task_map; pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */ pml4_entry_t *pm_pml4; /* VKA of top level */ vm_object_t pm_obj_pdpt; /* holds pdpt pages */ vm_object_t pm_obj_pml4; /* holds pml4 pages */ - vm_object_t pm_obj_top; /* holds single top level page */ +#define PMAP_PCID_MAX_CPUS (48) /* Must be a multiple of 8 */ + pcid_t pmap_pcid_cpus[PMAP_PCID_MAX_CPUS]; + volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS]; + struct pmap_statistics stats; /* map statistics */ + int ref_count; /* reference count */ + int nx_enabled; }; +#if NCOPY_WINDOWS > 0 #define PMAP_PDPT_FIRST_WINDOW 0 #define PMAP_PDPT_NWINDOWS 4 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS) @@ -431,6 +575,8 @@ typedef struct cpu_pmap { extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry); +extern void pmap_put_mapwindow(mapwindow_t *map); +#endif typedef struct pmap_memory_regions { ppnum_t base; @@ -439,16 +585,38 @@ typedef struct pmap_memory_regions { uint32_t type; } pmap_memory_region_t; -unsigned pmap_memory_region_count; -unsigned pmap_memory_region_current; +extern unsigned pmap_memory_region_count; +extern unsigned pmap_memory_region_current; #define PMAP_MEMORY_REGIONS_SIZE 128 extern pmap_memory_region_t pmap_memory_regions[]; +#include -static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) { - current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3); - current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT; +static inline void +set_dirbase(pmap_t tpmap, __unused thread_t thread) { + int ccpu = cpu_number(); + cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3; + cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map; +#ifndef __i386__ + /* + * Switch cr3 if necessary + * - unless running with no_shared_cr3 debugging mode + * and we're not on the kernel's cr3 (after pre-empted copyio) + */ + if (__probable(!no_shared_cr3)) { + if (get_cr3_base() != tpmap->pm_cr3) { + if (pmap_pcid_ncpus) { + pmap_pcid_activate(tpmap, ccpu); + } + else + set_cr3_raw(tpmap->pm_cr3); + } + } else { + if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3) + set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3); + } +#endif } /* @@ -465,6 +633,10 @@ extern void pmap_update_interrupt(void); extern addr64_t (kvtophys)( vm_offset_t addr); +extern void pmap_expand( + pmap_t pmap, + vm_map_offset_t addr); +#if !defined(__x86_64__) extern pt_entry_t *pmap_pte( struct pmap *pmap, vm_map_offset_t addr); @@ -480,7 +652,7 @@ extern pd_entry_t *pmap64_pde( extern pdpt_entry_t *pmap64_pdpt( struct pmap *pmap, vm_map_offset_t addr); - +#endif extern vm_offset_t pmap_map( vm_offset_t virt, vm_map_offset_t start, @@ -506,7 +678,8 @@ extern int pmap_list_resident_pages( struct pmap *pmap, vm_offset_t *listp, int space); - +extern void x86_filter_TLB_coherency_interrupts(boolean_t); +#ifdef __i386__ extern void pmap_commpage32_init( vm_offset_t kernel, vm_offset_t user, @@ -516,10 +689,17 @@ extern void pmap_commpage64_init( vm_map_offset_t user, int count); +#endif +/* + * Get cache attributes (as pagetable bits) for the specified phys page + */ +extern unsigned pmap_get_cache_attributes(ppnum_t); +#if NCOPY_WINDOWS > 0 extern struct cpu_pmap *pmap_cpu_alloc( boolean_t is_boot_cpu); extern void pmap_cpu_free( struct cpu_pmap *cp); +#endif extern void pmap_map_block( pmap_t pmap, @@ -533,14 +713,10 @@ extern void pmap_map_block( extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); -extern void pmap_sync_page_data_phys(ppnum_t pa); -extern void pmap_sync_page_attributes_phys(ppnum_t pa); -extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size); -extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr); -extern void pmap_map_sharedpage(task_t task, pmap_t pmap); -extern void pmap_unmap_sharedpage(pmap_t pmap); +extern void pmap_cpu_init(void); extern void pmap_disable_NX(pmap_t pmap); +#ifdef __i386__ extern void pmap_set_4GB_pagezero(pmap_t pmap); extern void pmap_clear_4GB_pagezero(pmap_t pmap); extern void pmap_load_kernel_cr3(void); @@ -549,10 +725,12 @@ extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types); extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types); extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int); extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int); +#endif -extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *); - - +extern void pt_fake_zone_init(int); +extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, + uint64_t *, int *, int *, int *); +extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2)); /* * Macros for speed. @@ -561,51 +739,26 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s #include -#if defined(PMAP_ACTIVATE_KERNEL) -#undef PMAP_ACTIVATE_KERNEL -#undef PMAP_DEACTIVATE_KERNEL -#undef PMAP_ACTIVATE_USER -#undef PMAP_DEACTIVATE_USER -#endif - - -#define PMAP_ACTIVATE_KERNEL(my_cpu) { \ - spl_t spl; \ - \ - spl = splhigh(); \ - if (current_cpu_datap()->cpu_tlb_invalid) \ - process_pmap_updates(); \ - splx(spl); \ -} - -#define PMAP_DEACTIVATE_KERNEL(my_cpu) { \ - spl_t spl; \ - \ - spl = splhigh(); \ - process_pmap_updates(); \ - splx(spl); \ -} - - -#define PMAP_ACTIVATE_MAP(map, my_cpu) { \ + +#define PMAP_ACTIVATE_MAP(map, thread) { \ register pmap_t tpmap; \ \ tpmap = vm_map_pmap(map); \ - set_dirbase(tpmap, my_cpu); \ + set_dirbase(tpmap, thread); \ } -#define PMAP_DEACTIVATE_MAP(map, my_cpu) - -#define PMAP_ACTIVATE_USER(th, my_cpu) { \ - spl_t spl; \ - \ - spl = splhigh(); \ - PMAP_ACTIVATE_MAP(th->map, my_cpu) \ - splx(spl); \ -} - -#define PMAP_DEACTIVATE_USER(th, my_cpu) +#ifdef __i386__ +#define PMAP_DEACTIVATE_MAP(map, thread) \ + if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \ + pmap_load_kernel_cr3(); +#elif defined(__x86_64__) +#define PMAP_DEACTIVATE_MAP(map, thread) \ + pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, cpu_number()) == (get_cr3_raw() & 0xFFF)) : TRUE); +#else +#define PMAP_DEACTIVATE_MAP(map, thread) +#endif +#if defined(__i386__) #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \ spl_t spl; \ @@ -616,16 +769,16 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s \ need_flush = 0; \ spl = splhigh(); \ - if (old_th->map != new_th->map) { \ - PMAP_DEACTIVATE_MAP(old_th->map, my_cpu); \ - PMAP_ACTIVATE_MAP(new_th->map, my_cpu); \ + if ((old_th->map != new_th->map) || (new_th->task != old_th->task)) { \ + PMAP_DEACTIVATE_MAP(old_th->map, old_th); \ + PMAP_ACTIVATE_MAP(new_th->map, new_th); \ } \ kpdp = current_cpu_datap()->cpu_copywindow_pdp; \ for (i = 0; i < NCOPY_WINDOWS; i++) { \ if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \ updp = pmap_pde(new_th->map->pmap, \ new_th->machine.copy_window[i].user_base);\ - *kpdp = updp ? *updp : 0; \ + pmap_store_pte(kpdp, updp ? *updp : 0); \ } \ kpdp++; \ } \ @@ -635,8 +788,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s else \ new_th->machine.copyio_state = WINDOWS_DIRTY; \ if (new_th->machine.physwindow_pte) { \ - *(current_cpu_datap()->cpu_physwindow_ptep) = \ - new_th->machine.physwindow_pte; \ + pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \ + new_th->machine.physwindow_pte); \ if (need_flush == 0) \ invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\ } \ @@ -644,22 +797,45 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s flush_tlb(); \ } +#else /* __x86_64__ */ +#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \ + \ + pmap_assert(ml_get_interrupts_enabled() == FALSE); \ + if (old_th->map != new_th->map) { \ + PMAP_DEACTIVATE_MAP(old_th->map, old_th); \ + PMAP_ACTIVATE_MAP(new_th->map, new_th); \ + } \ +} +#endif /* __i386__ */ + +#if NCOPY_WINDOWS > 0 +#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ + spl_t spl; \ + \ + spl = splhigh(); \ + PMAP_DEACTIVATE_MAP(th->map, th); \ + th->map = new_map; \ + PMAP_ACTIVATE_MAP(th->map, th); \ + splx(spl); \ + inval_copy_windows(th); \ +} +#else #define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ spl_t spl; \ \ spl = splhigh(); \ - PMAP_DEACTIVATE_MAP(th->map, my_cpu); \ + PMAP_DEACTIVATE_MAP(th->map, th); \ th->map = new_map; \ - PMAP_ACTIVATE_MAP(th->map, my_cpu); \ + PMAP_ACTIVATE_MAP(th->map, th); \ splx(spl); \ - inval_copy_windows(th); \ } +#endif /* * Marking the current cpu's cr3 inactive is achieved by setting its lsb. * Marking the current cpu's cr3 active once more involves clearng this bit. * Note that valid page tables are page-aligned and so the bottom 12 bits - * are noramlly zero. + * are normally zero, modulo PCID. * We can only mark the current cpu active/inactive but we can test any cpu. */ #define CPU_CR3_MARK_INACTIVE() \ @@ -671,7 +847,28 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s #define CPU_CR3_IS_ACTIVE(cpu) \ ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0) +#define CPU_GET_ACTIVE_CR3(cpu) \ + (cpu_datap(cpu)->cpu_active_cr3 & ~1) + +#define CPU_GET_TASK_CR3(cpu) \ + (cpu_datap(cpu)->cpu_task_cr3) + +/* + * Mark this cpu idle, and remove it from the active set, + * since it is not actively using any pmap. Signal_cpus + * will notice that it is idle, and avoid signaling it, + * but will queue the update request for when the cpu + * becomes active. + */ +#if defined(__x86_64__) #define MARK_CPU_IDLE(my_cpu) { \ + assert(ml_get_interrupts_enabled() == FALSE); \ + CPU_CR3_MARK_INACTIVE(); \ + __asm__ volatile("mfence"); \ +} +#else /* __i386__ native */ +#define MARK_CPU_IDLE(my_cpu) { \ + assert(ml_get_interrupts_enabled() == FALSE); \ /* \ * Mark this cpu idle, and remove it from the active set, \ * since it is not actively using any pmap. Signal_cpus \ @@ -679,19 +876,17 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s * but will queue the update request for when the cpu \ * becomes active. \ */ \ - int s = splhigh(); \ if (!cpu_mode_is64bit() || no_shared_cr3) \ process_pmap_updates(); \ else \ pmap_load_kernel_cr3(); \ CPU_CR3_MARK_INACTIVE(); \ __asm__ volatile("mfence"); \ - splx(s); \ } +#endif /* __i386__ */ #define MARK_CPU_ACTIVE(my_cpu) { \ - \ - int s = splhigh(); \ + assert(ml_get_interrupts_enabled() == FALSE); \ /* \ * If a kernel_pmap update was requested while this cpu \ * was idle, process it as if we got the interrupt. \ @@ -707,7 +902,6 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s \ if (current_cpu_datap()->cpu_tlb_invalid) \ process_pmap_updates(); \ - splx(s); \ } #define PMAP_CONTEXT(pmap, thread) @@ -718,12 +912,21 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s #define pmap_resident_count(pmap) ((pmap)->stats.resident_count) +#define pmap_resident_max(pmap) ((pmap)->stats.resident_max) #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr) #define pmap_attribute(pmap,addr,size,attr,value) \ (KERN_INVALID_ADDRESS) #define pmap_attribute_cache_sync(addr,size,attr,value) \ (KERN_INVALID_ADDRESS) +#define MACHINE_PMAP_IS_EMPTY 1 +extern boolean_t pmap_is_empty(pmap_t pmap, + vm_map_offset_t start, + vm_map_offset_t end); + +#define MACHINE_BOOTSTRAPPTD 1 /* Static bootstrap page-tables */ + + #endif /* ASSEMBLER */