-
/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
*
* Machine-dependent structures for the physical map module.
*/
-#ifdef KERNEL_PRIVATE
+
#ifndef _PMAP_MACHINE_
#define _PMAP_MACHINE_ 1
#include <kern/kern_types.h>
#include <kern/thread.h>
#include <kern/lock.h>
-
-#include <i386/mp.h>
-#include <i386/proc_reg.h>
+#define PMAP_QUEUE 1
+#ifdef PMAP_QUEUE
+#include <kern/queue.h>
+#endif
/*
* Define the generic in terms of the specific
* i386/i486/i860 Page Table Entry
*/
+#ifdef PAE
+typedef uint64_t pdpt_entry_t;
+typedef uint64_t pt_entry_t;
+typedef uint64_t pd_entry_t;
+typedef uint64_t pmap_paddr_t;
+#else
+typedef uint32_t pt_entry_t;
+typedef uint32_t pd_entry_t;
+typedef uint32_t pmap_paddr_t;
+#endif
+
+#define PT_ENTRY_NULL ((pt_entry_t *) 0)
+#define PD_ENTRY_NULL ((pt_entry_t *) 0)
+
#endif /* ASSEMBLER */
+#ifdef PAE
#define NPGPTD 4
#define PDESHIFT 21
#define PTEMASK 0x1ff
#define PTEINDX 3
-
+#else
+#define NPGPTD 1
+#define PDESHIFT 22
+#define PTEMASK 0x3ff
+#define PTEINDX 2
+#endif
#define PTESHIFT 12
#define PDESIZE sizeof(pd_entry_t) /* for assembly files */
#define PTESIZE sizeof(pt_entry_t) /* for assembly files */
#define INTEL_OFFMASK (I386_PGBYTES - 1)
-#define PG_FRAME 0x000FFFFFFFFFF000ULL
+#define PG_FRAME (~((pmap_paddr_t)PAGE_MASK))
#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
-#define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
#define NBPTD (NPGPTD << PAGE_SHIFT)
#define NPDEPTD (NBPTD / (sizeof (pd_entry_t)))
#define NBPDE (1 << PDESHIFT)
#define PDEMASK (NBPDE - 1)
- /* cleanly define parameters for all the page table levels */
-typedef uint64_t pml4_entry_t;
-#define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
-#define PML4SHIFT 39
-#define PML4PGSHIFT 9
-#define NBPML4 (1ULL << PML4SHIFT)
-#define PML4MASK (NBPML4-1)
-#define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
-
-typedef uint64_t pdpt_entry_t;
-#define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
-#define PDPTSHIFT 30
-#define PDPTPGSHIFT 9
-#define NBPDPT (1 << PDPTSHIFT)
-#define PDPTMASK (NBPDPT-1)
-#define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
-
-typedef uint64_t pd_entry_t;
-#define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
-#define PDSHIFT 21
-#define PDPGSHIFT 9
-#define NBPD (1 << PDSHIFT)
-#define PDMASK (NBPD-1)
-#define PD_ENTRY_NULL ((pd_entry_t *) 0)
-
-typedef uint64_t pt_entry_t;
-#define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
-#define PTSHIFT 12
-#define PTPGSHIFT 9
-#define NBPT (1 << PTSHIFT)
-#define PTMASK (NBPT-1)
-#define PT_ENTRY_NULL ((pt_entry_t *) 0)
-
-typedef uint64_t pmap_paddr_t;
-
-/*
- * Atomic 64-bit store of a page table entry.
- */
-static inline void
-pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
-{
- /*
- * Load the new value into %ecx:%ebx
- * Load the old value into %edx:%eax
- * Compare-exchange-8bytes at address entryp (loaded in %edi)
- * If the compare succeeds, the new value will have been stored.
- * Otherwise, the old value changed and reloaded, so try again.
- */
- asm volatile(
- " movl (%0), %%eax \n\t"
- " movl 4(%0), %%edx \n\t"
- "1: \n\t"
- " cmpxchg8b (%0) \n\t"
- " jnz 1b"
- :
- : "D" (entryp),
- "b" ((uint32_t)value),
- "c" ((uint32_t)(value >> 32))
- : "eax", "edx", "memory");
-}
-
-/* in 64 bit spaces, the number of each type of page in the page tables */
-#define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
-#define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
-#define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
-#define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
-
-/*
- * The 64-bit kernel is remapped in uber-space which is at the base
- * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
- * 512GB from the top of virtual space (or zero).
- */
-#define KERNEL_UBER_PML4_INDEX 511
-#define KERNEL_UBER_BASE (0ULL - NBPML4)
-#define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32))
-
#define VM_WIMG_COPYBACK VM_MEM_COHERENT
#define VM_WIMG_DEFAULT VM_MEM_COHERENT
/* ?? intel ?? */
/* write combining mode, aka store gather */
#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
-/*
- * Pte related macros
- */
-#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
-#define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
-
/*
* Size of Kernel address space. This is the number of page table pages
* (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
* This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
*/
#ifndef KVA_PAGES
-#define KVA_PAGES 1024
+#define KVA_PAGES 256
#endif
+/*
+ * Pte related macros
+ */
+#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
+
#ifndef NKPT
+#ifdef PAE
#define NKPT 500 /* actual number of kernel page tables */
+#else
+#define NKPT 32 /* initial number of kernel page tables */
+#endif
#endif
#ifndef NKPDE
#define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */
#endif
-
-enum high_cpu_types {
- HIGH_CPU_ISS0,
- HIGH_CPU_ISS1,
- HIGH_CPU_DESC,
- HIGH_CPU_LDT_BEGIN,
- HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
- HIGH_CPU_END
-};
-
-enum high_fixed_addresses {
- HIGH_FIXED_TRAMPS, /* must be first */
- HIGH_FIXED_TRAMPS_END,
- HIGH_FIXED_GDT,
- HIGH_FIXED_IDT,
- HIGH_FIXED_LDT_BEGIN,
- HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
- HIGH_FIXED_KTSS,
- HIGH_FIXED_DFTSS,
- HIGH_FIXED_DBTSS,
- HIGH_FIXED_CPUS_BEGIN,
- HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
-};
-
-
-/* XXX64 below PTDI values need cleanup */
/*
* The *PTDI values control the layout of virtual memory
*
*/
-#define KPTDI (0x000)/* start of kernel virtual pde's */
+#ifdef PAE
+#define KPTDI (0x600)/* start of kernel virtual pde's */
#define PTDPTDI (0x7F4) /* ptd entry that points to ptd! */
#define APTDPTDI (0x7F8) /* alt ptd entry that points to APTD */
-#define UMAXPTDI (0x7F8) /* ptd entry for user space end */
+#define UMAXPTDI (0x5FC) /* ptd entry for user space end */
#define UMAXPTEOFF (NPTEPG) /* pte entry for user space end */
+#else
+#define KPTDI (0x300)/* start of kernel virtual pde's */
+#define PTDPTDI (0x3FD) /* ptd entry that points to ptd! */
+#define APTDPTDI (0x3FE) /* alt ptd entry that points to APTD */
+#define UMAXPTDI (0x2FF) /* ptd entry for user space end */
+#define UMAXPTEOFF (NPTEPG) /* pte entry for user space end */
+#endif
#define KERNBASE VADDR(KPTDI,0)
-/*
- * Convert address offset to directory address
- * containing the page table pointer - legacy
- */
-/*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
-
-#define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */
-#define pmap_index_to_virt(x) (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
-
/*
* Convert address offset to page descriptor index
*/
-#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
+#define pdenum(pmap, a) (((a) >> PDESHIFT) & PDEMASK)
-#define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1))
-#define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
-#define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
/*
* Convert page descriptor index to user virtual address
/*
* Convert address offset to page table index
*/
-#define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
+#define ptenum(a) (((a) >> PTESHIFT) & PTEMASK)
/*
* Hardware pte bit definitions (to be used directly on the ptes
#define INTEL_PTE_PS 0x00000080
#define INTEL_PTE_GLOBAL 0x00000100
#define INTEL_PTE_WIRED 0x00000200
-#define INTEL_PTE_PFN PG_FRAME
+#define INTEL_PTE_PFN /*0xFFFFF000*/ (~0xFFF)
#define INTEL_PTE_PTA 0x00000080
-#define INTEL_PTE_NX (1ULL << 63)
-
-#define INTEL_PTE_INVALID 0
-
#define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */
#define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */
#define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1)
-#define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
-#define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
-#define pte_user_rw(p) ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
-#define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
-
#define PMAP_DEFAULT_CACHE 0
#define PMAP_INHIBIT_CACHE 1
#define PMAP_GUARDED_CACHE 2
extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */
+#ifdef PAE
extern pdpt_entry_t *IdlePDPT;
-
-extern pmap_paddr_t lo_kernel_cr3;
-
-extern pml4_entry_t *IdlePML4;
-extern pdpt_entry_t *IdlePDPT64;
-extern addr64_t kernel64_cr3;
-extern boolean_t no_shared_cr3;
+#endif
/*
* virtual address to page table entry and
* Note: these work recursively, thus vtopte of a pte will give
* the corresponding pde that in turn maps it.
*/
-#define vtopte(va) (PTmap + i386_btop((vm_offset_t)va))
+#define vtopte(va) (PTmap + i386_btop(va))
typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */
*/
struct pmap {
- pd_entry_t *dirbase; /* page directory pointer */
- pmap_paddr_t pdirbase; /* phys. address of dirbase */
- vm_object_t pm_obj; /* object to hold pde's */
+#ifdef PMAP_QUEUE
+ queue_head_t pmap_link; /* unordered queue of in use pmaps */
+#endif
+ pd_entry_t *dirbase; /* page directory pointer register */
+ pd_entry_t *pdirbase; /* phys. address of dirbase */
+ vm_object_t pm_obj; /* object to hold pte's */
int ref_count; /* reference count */
- int nx_enabled;
- boolean_t pm_64bit;
- boolean_t pm_kernel_cr3;
decl_simple_lock_data(,lock) /* lock on map */
struct pmap_statistics stats; /* map statistics */
+ cpu_set cpus_using; /* bitmap of cpus using pmap */
+#ifdef PAE
vm_offset_t pm_hold; /* true pdpt zalloc addr */
- pmap_paddr_t pm_cr3; /* physical addr */
- pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */
- pml4_entry_t *pm_pml4; /* VKA of top level */
- vm_object_t pm_obj_pdpt; /* holds pdpt pages */
- vm_object_t pm_obj_pml4; /* holds pml4 pages */
- vm_object_t pm_obj_top; /* holds single top level page */
+ pdpt_entry_t *pm_pdpt; /* KVA of pg dir ptr table */
+ vm_offset_t pm_ppdpt; /* phy addr pdpt
+ should really be 32/64 bit */
+#endif
};
-
-#define PMAP_PDPT_FIRST_WINDOW 0
-#define PMAP_PDPT_NWINDOWS 4
-#define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
-#define PMAP_PDE_NWINDOWS 4
-#define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
-#define PMAP_PTE_NWINDOWS 4
-
-#define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
-#define PMAP_WINDOW_SIZE 8
-#define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
-
+#define PMAP_NWINDOWS 4
typedef struct {
pt_entry_t *prv_CMAP;
caddr_t prv_CADDR;
} mapwindow_t;
typedef struct cpu_pmap {
- int pdpt_window_index;
- int pde_window_index;
- int pte_window_index;
mapwindow_t mapwindow[PMAP_NWINDOWS];
+ struct pmap *real_pmap;
+ struct pmap_update_list *update_list;
+ volatile boolean_t update_needed;
} cpu_pmap_t;
-
-extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
+/*
+ * Should be rewritten in asm anyway.
+ */
+#define CM1 (current_cpu_datap()->cpu_pmap->mapwindow[0].prv_CMAP)
+#define CM2 (current_cpu_datap()->cpu_pmap->mapwindow[1].prv_CMAP)
+#define CM3 (current_cpu_datap()->cpu_pmap->mapwindow[2].prv_CMAP)
+#define CM4 (current_cpu_datap()->cpu_pmap->mapwindow[3].prv_CMAP)
+#define CA1 (current_cpu_datap()->cpu_pmap->mapwindow[0].prv_CADDR)
+#define CA2 (current_cpu_datap()->cpu_pmap->mapwindow[1].prv_CADDR)
+#define CA3 (current_cpu_datap()->cpu_pmap->mapwindow[2].prv_CADDR)
+#define CA4 (current_cpu_datap()->cpu_pmap->mapwindow[3].prv_CADDR)
typedef struct pmap_memory_regions {
ppnum_t base;
unsigned pmap_memory_region_count;
unsigned pmap_memory_region_current;
-#define PMAP_MEMORY_REGIONS_SIZE 128
+#define PMAP_MEMORY_REGIONS_SIZE 32
extern pmap_memory_region_t pmap_memory_regions[];
-static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
- current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
- current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
+/*
+ * Optimization avoiding some TLB flushes when switching to
+ * kernel-loaded threads. This is effective only for i386:
+ * Since user task, kernel task and kernel loaded tasks share the
+ * same virtual space (with appropriate protections), any pmap
+ * allows mapping kernel and kernel loaded tasks.
+ *
+ * The idea is to avoid switching to another pmap unnecessarily when
+ * switching to a kernel-loaded task, or when switching to the kernel
+ * itself.
+ *
+ * We store the pmap we are really using (from which we fetched the
+ * dirbase value) in current_cpu_datap()->cpu_pmap.real_pmap.
+ *
+ * Invariant:
+ * current_pmap() == current_cpu_datap()->cpu_pmap.real_pmap ||
+ * current_pmap() == kernel_pmap.
+ */
+#define PMAP_REAL(my_cpu) (cpu_datap(my_cpu)->cpu_pmap->real_pmap)
+
+#include <i386/proc_reg.h>
+/*
+ * If switching to the kernel pmap, don't incur the TLB cost of switching
+ * to its page tables, since all maps include the kernel map as a subset.
+ * Simply record that this CPU is logically on the kernel pmap (see
+ * pmap_destroy).
+ *
+ * Similarly, if switching to a pmap (other than kernel_pmap that is already
+ * in use, don't do anything to the hardware, to avoid a TLB flush.
+ */
+
+#define PMAP_CPU_SET(pmap, my_cpu) i_bit_set(my_cpu, &((pmap)->cpus_using))
+#define PMAP_CPU_CLR(pmap, my_cpu) i_bit_clear(my_cpu, &((pmap)->cpus_using))
+
+#ifdef PAE
+#define PDIRBASE pm_ppdpt
+#else
+#define PDIRBASE pdirbase
+#endif
+#define set_dirbase(mypmap, my_cpu) { \
+ struct pmap **ppmap = &PMAP_REAL(my_cpu); \
+ pmap_paddr_t pdirbase = (pmap_paddr_t)((mypmap)->PDIRBASE); \
+ \
+ if (*ppmap == (pmap_paddr_t)NULL) { \
+ *ppmap = (mypmap); \
+ PMAP_CPU_SET((mypmap), my_cpu); \
+ set_cr3(pdirbase); \
+ } else if ((mypmap) != kernel_pmap && (mypmap) != *ppmap ) { \
+ if (*ppmap != kernel_pmap) \
+ PMAP_CPU_CLR(*ppmap, my_cpu); \
+ *ppmap = (mypmap); \
+ PMAP_CPU_SET((mypmap), my_cpu); \
+ set_cr3(pdirbase); \
+ } \
+ assert((mypmap) == *ppmap || (mypmap) == kernel_pmap); \
}
+/*
+ * List of cpus that are actively using mapped memory. Any
+ * pmap update operation must wait for all cpus in this list.
+ * Update operations must still be queued to cpus not in this
+ * list.
+ */
+extern cpu_set cpus_active;
+
+/*
+ * List of cpus that are idle, but still operating, and will want
+ * to see any kernel pmap updates when they become active.
+ */
+extern cpu_set cpus_idle;
+
+
+#define cpu_update_needed(cpu) cpu_datap(cpu)->cpu_pmap->update_needed
+#define cpu_update_list(cpu) cpu_datap(cpu)->cpu_pmap->update_list
+
/*
* External declarations for PMAP_ACTIVATE.
*/
-extern void process_pmap_updates(void);
+extern void process_pmap_updates(struct pmap *pmap);
extern void pmap_update_interrupt(void);
/*
* Machine dependent routines that are used only for i386/i486/i860.
*/
-extern addr64_t (kvtophys)(
+extern vm_offset_t (kvtophys)(
vm_offset_t addr);
extern pt_entry_t *pmap_pte(
struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pd_entry_t *pmap_pde(
- struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pd_entry_t *pmap64_pde(
- struct pmap *pmap,
- vm_map_offset_t addr);
-
-extern pdpt_entry_t *pmap64_pdpt(
- struct pmap *pmap,
- vm_map_offset_t addr);
+ vm_offset_t addr);
extern vm_offset_t pmap_map(
vm_offset_t virt,
- vm_map_offset_t start,
- vm_map_offset_t end,
- vm_prot_t prot,
- unsigned int flags);
+ vm_offset_t start,
+ vm_offset_t end,
+ vm_prot_t prot);
extern vm_offset_t pmap_map_bd(
vm_offset_t virt,
- vm_map_offset_t start,
- vm_map_offset_t end,
- vm_prot_t prot,
- unsigned int flags);
+ vm_offset_t start,
+ vm_offset_t end,
+ vm_prot_t prot);
extern void pmap_bootstrap(
- vm_offset_t load_start,
- boolean_t IA32e);
+ vm_offset_t load_start);
extern boolean_t pmap_valid_page(
ppnum_t pn);
vm_offset_t *listp,
int space);
-extern void pmap_commpage32_init(
+extern void pmap_commpage_init(
vm_offset_t kernel,
vm_offset_t user,
int count);
-extern void pmap_commpage64_init(
- vm_offset_t kernel,
- vm_map_offset_t user,
- int count);
-
extern struct cpu_pmap *pmap_cpu_alloc(
boolean_t is_boot_cpu);
extern void pmap_cpu_free(
struct cpu_pmap *cp);
-
-extern void pmap_map_block(
- pmap_t pmap,
- addr64_t va,
- ppnum_t pa,
- uint32_t size,
- vm_prot_t prot,
- int attr,
- unsigned int flags);
extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
extern void pmap_sync_page_data_phys(ppnum_t pa);
extern void pmap_sync_page_attributes_phys(ppnum_t pa);
-extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
-extern void pmap_disable_NX(pmap_t pmap);
-extern void pmap_set_4GB_pagezero(pmap_t pmap);
-extern void pmap_clear_4GB_pagezero(pmap_t pmap);
-extern void pmap_load_kernel_cr3(void);
-extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
-extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
-extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
-extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
-extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
-
-extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
-
-
-
/*
* Macros for speed.
*/
#undef PMAP_DEACTIVATE_USER
#endif
+/*
+ * For multiple CPUS, PMAP_ACTIVATE and PMAP_DEACTIVATE must manage
+ * fields to control TLB invalidation on other CPUS.
+ */
-#define PMAP_ACTIVATE_KERNEL(my_cpu) { \
- spl_t spl; \
- \
- spl = splhigh(); \
- if (current_cpu_datap()->cpu_tlb_invalid) \
- process_pmap_updates(); \
- splx(spl); \
+#define PMAP_ACTIVATE_KERNEL(my_cpu) { \
+ \
+ /* \
+ * Let pmap updates proceed while we wait for this pmap. \
+ */ \
+ i_bit_clear((my_cpu), &cpus_active); \
+ \
+ /* \
+ * Lock the pmap to put this cpu in its active set. \
+ * Wait for updates here. \
+ */ \
+ simple_lock(&kernel_pmap->lock); \
+ \
+ /* \
+ * Process invalidate requests for the kernel pmap. \
+ */ \
+ if (cpu_update_needed(my_cpu)) \
+ process_pmap_updates(kernel_pmap); \
+ \
+ /* \
+ * Mark that this cpu is using the pmap. \
+ */ \
+ i_bit_set((my_cpu), &kernel_pmap->cpus_using); \
+ \
+ /* \
+ * Mark this cpu active - IPL will be lowered by \
+ * load_context(). \
+ */ \
+ i_bit_set((my_cpu), &cpus_active); \
+ \
+ simple_unlock(&kernel_pmap->lock); \
}
-#define PMAP_DEACTIVATE_KERNEL(my_cpu) { \
- spl_t spl; \
- \
- spl = splhigh(); \
- process_pmap_updates(); \
- splx(spl); \
+#define PMAP_DEACTIVATE_KERNEL(my_cpu) { \
+ /* \
+ * Mark pmap no longer in use by this cpu even if \
+ * pmap is locked against updates. \
+ */ \
+ i_bit_clear((my_cpu), &kernel_pmap->cpus_using); \
+ i_bit_clear((my_cpu), &cpus_active); \
+ PMAP_REAL(my_cpu) = NULL; \
}
-
#define PMAP_ACTIVATE_MAP(map, my_cpu) { \
register pmap_t tpmap; \
- \
- tpmap = vm_map_pmap(map); \
- set_dirbase(tpmap, my_cpu); \
+ \
+ tpmap = vm_map_pmap(map); \
+ if (tpmap == kernel_pmap) { \
+ /* \
+ * If this is the kernel pmap, switch to its page tables. \
+ */ \
+ set_dirbase(kernel_pmap, my_cpu); \
+ } \
+ else { \
+ /* \
+ * Let pmap updates proceed while we wait for this pmap. \
+ */ \
+ i_bit_clear((my_cpu), &cpus_active); \
+ \
+ /* \
+ * Lock the pmap to put this cpu in its active set. \
+ * Wait for updates here. \
+ */ \
+ simple_lock(&tpmap->lock); \
+ \
+ /* \
+ * No need to invalidate the TLB - the entire user pmap \
+ * will be invalidated by reloading dirbase. \
+ */ \
+ set_dirbase(tpmap, my_cpu); \
+ \
+ /* \
+ * Mark this cpu active - IPL will be lowered by \
+ * load_context(). \
+ */ \
+ i_bit_set((my_cpu), &cpus_active); \
+ \
+ simple_unlock(&tpmap->lock); \
+ } \
}
#define PMAP_DEACTIVATE_MAP(map, my_cpu)
-#define PMAP_ACTIVATE_USER(th, my_cpu) { \
- spl_t spl; \
- \
- spl = splhigh(); \
- PMAP_ACTIVATE_MAP(th->map, my_cpu) \
- splx(spl); \
+#define PMAP_ACTIVATE_USER(th, my_cpu) { \
+ spl_t spl; \
+ \
+ spl = splhigh(); \
+ PMAP_ACTIVATE_MAP(th->map, my_cpu) \
+ splx(spl); \
}
#define PMAP_DEACTIVATE_USER(th, my_cpu)
-
#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \
spl_t spl; \
- pt_entry_t *kpdp; \
- pt_entry_t *updp; \
- int i; \
- int need_flush; \
- \
- need_flush = 0; \
- spl = splhigh(); \
+ \
if (old_th->map != new_th->map) { \
+ spl = splhigh(); \
PMAP_DEACTIVATE_MAP(old_th->map, my_cpu); \
PMAP_ACTIVATE_MAP(new_th->map, my_cpu); \
+ splx(spl); \
} \
- kpdp = current_cpu_datap()->cpu_copywindow_pdp; \
- for (i = 0; i < NCOPY_WINDOWS; i++) { \
- if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \
- updp = pmap_pde(new_th->map->pmap, \
- new_th->machine.copy_window[i].user_base);\
- *kpdp = updp ? *updp : 0; \
- } \
- kpdp++; \
- } \
- splx(spl); \
- if (new_th->machine.copyio_state == WINDOWS_OPENED) \
- need_flush = 1; \
- else \
- new_th->machine.copyio_state = WINDOWS_DIRTY; \
- if (new_th->machine.physwindow_pte) { \
- *(current_cpu_datap()->cpu_physwindow_ptep) = \
- new_th->machine.physwindow_pte; \
- if (need_flush == 0) \
- invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
- } \
- if (need_flush) \
- flush_tlb(); \
}
#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \
spl_t spl; \
\
- spl = splhigh(); \
+ spl = splhigh(); \
PMAP_DEACTIVATE_MAP(th->map, my_cpu); \
th->map = new_map; \
PMAP_ACTIVATE_MAP(th->map, my_cpu); \
splx(spl); \
- inval_copy_windows(th); \
}
-/*
- * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
- * Marking the current cpu's cr3 active once more involves clearng this bit.
- * Note that valid page tables are page-aligned and so the bottom 12 bits
- * are noramlly zero.
- * We can only mark the current cpu active/inactive but we can test any cpu.
- */
-#define CPU_CR3_MARK_INACTIVE() \
- current_cpu_datap()->cpu_active_cr3 |= 1
-
-#define CPU_CR3_MARK_ACTIVE() \
- current_cpu_datap()->cpu_active_cr3 &= ~1
-
-#define CPU_CR3_IS_ACTIVE(cpu) \
- ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
-
#define MARK_CPU_IDLE(my_cpu) { \
/* \
* Mark this cpu idle, and remove it from the active set, \
* becomes active. \
*/ \
int s = splhigh(); \
- if (!cpu_mode_is64bit() || no_shared_cr3) \
- process_pmap_updates(); \
- else \
- pmap_load_kernel_cr3(); \
- CPU_CR3_MARK_INACTIVE(); \
- __asm__ volatile("mfence"); \
+ i_bit_set((my_cpu), &cpus_idle); \
+ i_bit_clear((my_cpu), &cpus_active); \
splx(s); \
+ set_led(my_cpu); \
}
-#define MARK_CPU_ACTIVE(my_cpu) { \
+#define MARK_CPU_ACTIVE(my_cpu) { \
\
int s = splhigh(); \
/* \
* set assures that we will receive another update \
* interrupt if this happens. \
*/ \
- CPU_CR3_MARK_ACTIVE(); \
- __asm__ volatile("mfence"); \
+ i_bit_clear((my_cpu), &cpus_idle); \
+ \
+ if (cpu_update_needed(my_cpu)) \
+ pmap_update_interrupt(); \
\
- if (current_cpu_datap()->cpu_tlb_invalid) \
- process_pmap_updates(); \
+ /* \
+ * Mark that this cpu is now active. \
+ */ \
+ i_bit_set((my_cpu), &cpus_active); \
splx(s); \
+ clear_led(my_cpu); \
}
#define PMAP_CONTEXT(pmap, thread)
#define pmap_kernel_va(VA) \
- ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \
- (((vm_offset_t) (VA)) <= vm_max_kernel_address))
-
+ (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS))
#define pmap_resident_count(pmap) ((pmap)->stats.resident_count)
#define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
#endif /* ASSEMBLER */
-
#endif /* _PMAP_MACHINE_ */
-
-
-#endif /* KERNEL_PRIVATE */
projects / apple / xnu.git / blobdiff