X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4452a7af2eac33dbad800bcc91f2399d62c18f53..e2fac8b15b12a7979f72090454d850e612fc5b13:/osfmk/i386/pmap.h diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h index 8a50070d4..90f9ff8d7 100644 --- a/osfmk/i386/pmap.h +++ b/osfmk/i386/pmap.h @@ -1,6 +1,5 @@ - /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -174,7 +173,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) * If the compare succeeds, the new value will have been stored. * Otherwise, the old value changed and reloaded, so try again. */ - asm volatile( + __asm__ volatile( " movl (%0), %%eax \n\t" " movl 4(%0), %%edx \n\t" "1: \n\t" @@ -187,6 +186,39 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) : "eax", "edx", "memory"); } +/* + * Atomic 64-bit compare and exchange of a page table entry. + */ +static inline boolean_t +pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new) +{ + boolean_t ret; + + /* + * Load the old value into %edx:%eax + * Load the new value into %ecx:%ebx + * Compare-exchange-8bytes at address entryp (loaded in %edi) + * If the compare succeeds, the new value is stored, return TRUE. + * Otherwise, no swap is made, return FALSE. + */ + asm volatile( + " lock; cmpxchg8b (%1) \n\t" + " setz %%al \n\t" + " movzbl %%al,%0" + : "=a" (ret) + : "D" (entryp), + "a" ((uint32_t)old), + "d" ((uint32_t)(old >> 32)), + "b" ((uint32_t)new), + "c" ((uint32_t)(new >> 32)) + : "memory"); + return ret; +} + +#define pmap_update_pte(entryp, old, new) \ + while (!pmap_cmpx_pte((entryp), (old), (new))) + + /* in 64 bit spaces, the number of each type of page in the page tables */ #define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t)))) #define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t)))) @@ -360,6 +392,10 @@ extern pdpt_entry_t *IdlePDPT64; extern addr64_t kernel64_cr3; extern boolean_t no_shared_cr3; +extern uint64_t pmap_pv_hashlist_walks; +extern uint64_t pmap_pv_hashlist_cnts; +extern uint32_t pmap_pv_hashlist_max; + /* * virtual address to page table entry and * to physical address. Likewise for alternate address space. @@ -390,9 +426,7 @@ struct pmap { vm_object_t pm_obj; /* object to hold pde's */ int ref_count; /* reference count */ int nx_enabled; - boolean_t pm_64bit; - boolean_t pm_kernel_cr3; - boolean_t pm_shared; + task_map_t pm_task_map; decl_simple_lock_data(,lock) /* lock on map */ struct pmap_statistics stats; /* map statistics */ vm_offset_t pm_hold; /* true pdpt zalloc addr */ @@ -402,6 +436,7 @@ struct pmap { vm_object_t pm_obj_pdpt; /* holds pdpt pages */ vm_object_t pm_obj_pml4; /* holds pml4 pages */ vm_object_t pm_obj_top; /* holds single top level page */ + boolean_t pm_shared; }; @@ -430,6 +465,8 @@ typedef struct cpu_pmap { extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry); +extern void pmap_put_mapwindow(mapwindow_t *map); + typedef struct pmap_memory_regions { ppnum_t base; @@ -447,7 +484,7 @@ extern pmap_memory_region_t pmap_memory_regions[]; static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) { current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3); - current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT; + current_cpu_datap()->cpu_task_map = tpmap->pm_task_map; } /* @@ -464,6 +501,10 @@ extern void pmap_update_interrupt(void); extern addr64_t (kvtophys)( vm_offset_t addr); +extern void pmap_expand( + pmap_t pmap, + vm_map_offset_t addr); + extern pt_entry_t *pmap_pte( struct pmap *pmap, vm_map_offset_t addr); @@ -532,13 +573,8 @@ extern void pmap_map_block( extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); -extern void pmap_sync_page_data_phys(ppnum_t pa); -extern void pmap_sync_page_attributes_phys(ppnum_t pa); -extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size); -extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr); -extern void pmap_map_sharedpage(task_t task, pmap_t pmap); -extern void pmap_unmap_sharedpage(pmap_t pmap); +extern void pmap_cpu_init(void); extern void pmap_disable_NX(pmap_t pmap); extern void pmap_set_4GB_pagezero(pmap_t pmap); extern void pmap_clear_4GB_pagezero(pmap_t pmap); @@ -593,9 +629,10 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s set_dirbase(tpmap, my_cpu); \ } -#define PMAP_DEACTIVATE_MAP(map, my_cpu) \ - if (current_cpu_datap()->cpu_task_map == TASK_MAP_64BIT_SHARED) \ - pmap_load_kernel_cr3(); +#define PMAP_DEACTIVATE_MAP(map, my_cpu) \ + if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \ + pmap_load_kernel_cr3(); + #define PMAP_ACTIVATE_USER(th, my_cpu) { \ spl_t spl; \ @@ -626,7 +663,7 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \ updp = pmap_pde(new_th->map->pmap, \ new_th->machine.copy_window[i].user_base);\ - pmap_store_pte(kpdp, updp ? *updp : 0); \ + pmap_store_pte(kpdp, updp ? *updp : 0); \ } \ kpdp++; \ } \ @@ -636,8 +673,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s else \ new_th->machine.copyio_state = WINDOWS_DIRTY; \ if (new_th->machine.physwindow_pte) { \ - pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \ - new_th->machine.physwindow_pte); \ + pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \ + new_th->machine.physwindow_pte); \ if (need_flush == 0) \ invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\ } \ @@ -672,8 +709,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s #define CPU_CR3_IS_ACTIVE(cpu) \ ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0) -#define CPU_GET_ACTIVE_CR3(cpu) \ - (cpu_datap(cpu)->cpu_active_cr3 & ~1) +#define CPU_GET_ACTIVE_CR3(cpu) \ + (cpu_datap(cpu)->cpu_active_cr3 & ~1) #define MARK_CPU_IDLE(my_cpu) { \ /* \ @@ -722,12 +759,18 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s #define pmap_resident_count(pmap) ((pmap)->stats.resident_count) +#define pmap_resident_max(pmap) ((pmap)->stats.resident_max) #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr) #define pmap_attribute(pmap,addr,size,attr,value) \ (KERN_INVALID_ADDRESS) #define pmap_attribute_cache_sync(addr,size,attr,value) \ (KERN_INVALID_ADDRESS) +#define MACHINE_PMAP_IS_EMPTY 1 +extern boolean_t pmap_is_empty(pmap_t pmap, + vm_map_offset_t start, + vm_map_offset_t end); + #endif /* ASSEMBLER */