-
/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the
- * License may not be used to create, or enable the creation or
- * redistribution of, unlawful or unlicensed copies of an Apple operating
- * system, or to circumvent, violate, or enable the circumvention or
- * violation of, any terms of an Apple operating system software license
- * agreement.
- *
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
* limitations under the License.
- *
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
* If the compare succeeds, the new value will have been stored.
* Otherwise, the old value changed and reloaded, so try again.
*/
- asm volatile(
+ __asm__ volatile(
" movl (%0), %%eax \n\t"
" movl 4(%0), %%edx \n\t"
"1: \n\t"
: "eax", "edx", "memory");
}
+/*
+ * Atomic 64-bit compare and exchange of a page table entry.
+ */
+static inline boolean_t
+pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
+{
+ boolean_t ret;
+
+ /*
+ * Load the old value into %edx:%eax
+ * Load the new value into %ecx:%ebx
+ * Compare-exchange-8bytes at address entryp (loaded in %edi)
+ * If the compare succeeds, the new value is stored, return TRUE.
+ * Otherwise, no swap is made, return FALSE.
+ */
+ asm volatile(
+ " lock; cmpxchg8b (%1) \n\t"
+ " setz %%al \n\t"
+ " movzbl %%al,%0"
+ : "=a" (ret)
+ : "D" (entryp),
+ "a" ((uint32_t)old),
+ "d" ((uint32_t)(old >> 32)),
+ "b" ((uint32_t)new),
+ "c" ((uint32_t)(new >> 32))
+ : "memory");
+ return ret;
+}
+
+#define pmap_update_pte(entryp, old, new) \
+ while (!pmap_cmpx_pte((entryp), (old), (new)))
+
+
/* in 64 bit spaces, the number of each type of page in the page tables */
#define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
#define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
extern addr64_t kernel64_cr3;
extern boolean_t no_shared_cr3;
+extern uint64_t pmap_pv_hashlist_walks;
+extern uint64_t pmap_pv_hashlist_cnts;
+extern uint32_t pmap_pv_hashlist_max;
+
/*
* virtual address to page table entry and
* to physical address. Likewise for alternate address space.
vm_object_t pm_obj; /* object to hold pde's */
int ref_count; /* reference count */
int nx_enabled;
- boolean_t pm_64bit;
- boolean_t pm_kernel_cr3;
+ task_map_t pm_task_map;
decl_simple_lock_data(,lock) /* lock on map */
struct pmap_statistics stats; /* map statistics */
vm_offset_t pm_hold; /* true pdpt zalloc addr */
vm_object_t pm_obj_pdpt; /* holds pdpt pages */
vm_object_t pm_obj_pml4; /* holds pml4 pages */
vm_object_t pm_obj_top; /* holds single top level page */
+ boolean_t pm_shared;
};
extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
+extern void pmap_put_mapwindow(mapwindow_t *map);
+
typedef struct pmap_memory_regions {
ppnum_t base;
static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
- current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
+ current_cpu_datap()->cpu_task_map = tpmap->pm_task_map;
}
/*
extern addr64_t (kvtophys)(
vm_offset_t addr);
+extern void pmap_expand(
+ pmap_t pmap,
+ vm_map_offset_t addr);
+
extern pt_entry_t *pmap_pte(
struct pmap *pmap,
vm_map_offset_t addr);
extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
-extern void pmap_sync_page_data_phys(ppnum_t pa);
-extern void pmap_sync_page_attributes_phys(ppnum_t pa);
-extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern void pmap_cpu_init(void);
extern void pmap_disable_NX(pmap_t pmap);
extern void pmap_set_4GB_pagezero(pmap_t pmap);
extern void pmap_clear_4GB_pagezero(pmap_t pmap);
set_dirbase(tpmap, my_cpu); \
}
-#define PMAP_DEACTIVATE_MAP(map, my_cpu)
+#define PMAP_DEACTIVATE_MAP(map, my_cpu) \
+ if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \
+ pmap_load_kernel_cr3();
+
#define PMAP_ACTIVATE_USER(th, my_cpu) { \
spl_t spl; \
if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \
updp = pmap_pde(new_th->map->pmap, \
new_th->machine.copy_window[i].user_base);\
- *kpdp = updp ? *updp : 0; \
+ pmap_store_pte(kpdp, updp ? *updp : 0); \
} \
kpdp++; \
} \
else \
new_th->machine.copyio_state = WINDOWS_DIRTY; \
if (new_th->machine.physwindow_pte) { \
- *(current_cpu_datap()->cpu_physwindow_ptep) = \
- new_th->machine.physwindow_pte; \
+ pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \
+ new_th->machine.physwindow_pte); \
if (need_flush == 0) \
invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
} \
#define CPU_CR3_IS_ACTIVE(cpu) \
((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
+#define CPU_GET_ACTIVE_CR3(cpu) \
+ (cpu_datap(cpu)->cpu_active_cr3 & ~1)
+
#define MARK_CPU_IDLE(my_cpu) { \
/* \
* Mark this cpu idle, and remove it from the active set, \
#define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
+#define pmap_resident_max(pmap) ((pmap)->stats.resident_max)
#define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
#define pmap_attribute(pmap,addr,size,attr,value) \
(KERN_INVALID_ADDRESS)
#define pmap_attribute_cache_sync(addr,size,attr,value) \
(KERN_INVALID_ADDRESS)
+#define MACHINE_PMAP_IS_EMPTY 1
+extern boolean_t pmap_is_empty(pmap_t pmap,
+ vm_map_offset_t start,
+ vm_map_offset_t end);
+
#endif /* ASSEMBLER */
projects / apple / xnu.git / blobdiff