+ address = (vm_offset_t)mapaddr;
+
+ for (i = 0; i < PMAP_NWINDOWS; i++, address += PAGE_SIZE) {
+ spl_t s;
+ s = splhigh();
+ while ((pte = pmap_pte(kernel_pmap, (vm_map_offset_t)address)) == 0)
+ pmap_expand(kernel_pmap, (vm_map_offset_t)address);
+ * (int *) pte = 0;
+ cp->mapwindow[i].prv_CADDR = (caddr_t) address;
+ cp->mapwindow[i].prv_CMAP = pte;
+ splx(s);
+ }
+ vm_map_unlock(kernel_map);
+ }
+
+ cp->pdpt_window_index = PMAP_PDPT_FIRST_WINDOW;
+ cp->pde_window_index = PMAP_PDE_FIRST_WINDOW;
+ cp->pte_window_index = PMAP_PTE_FIRST_WINDOW;
+
+ return cp;
+}
+
+void
+pmap_cpu_free(struct cpu_pmap *cp)
+{
+ if (cp != NULL && cp != &cpu_pmap_master) {
+ kfree((void *) cp, sizeof(cpu_pmap_t));
+ }
+}
+
+
+mapwindow_t *
+pmap_get_mapwindow(pt_entry_t pentry)
+{
+ mapwindow_t *mp;
+ int i;
+
+ assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+
+ /*
+ * Note: 0th map reserved for pmap_pte()
+ */
+ for (i = PMAP_NWINDOWS_FIRSTFREE; i < PMAP_NWINDOWS; i++) {
+ mp = ¤t_cpu_datap()->cpu_pmap->mapwindow[i];
+
+ if (*mp->prv_CMAP == 0) {
+ pmap_store_pte(mp->prv_CMAP, pentry);
+
+ invlpg((uintptr_t)mp->prv_CADDR);
+
+ return (mp);
+ }
+ }
+ panic("pmap_get_mapwindow: no windows available");
+
+ return NULL;
+}
+
+
+void
+pmap_put_mapwindow(mapwindow_t *mp)
+{
+ pmap_store_pte(mp->prv_CMAP, 0);
+}
+
+
+/*
+ * The Intel platform can nest at the PDE level, so NBPDE (i.e. 2MB) at a time,
+ * on a NBPDE boundary.
+ */
+uint64_t pmap_nesting_size_min = NBPDE;
+uint64_t pmap_nesting_size_max = 0 - (uint64_t)NBPDE; /* no limit, really... */
+
+/*
+ * kern_return_t pmap_nest(grand, subord, vstart, size)
+ *
+ * grand = the pmap that we will nest subord into
+ * subord = the pmap that goes into the grand
+ * vstart = start of range in pmap to be inserted
+ * nstart = start of range in pmap nested pmap
+ * size = Size of nest area (up to 16TB)
+ *
+ * Inserts a pmap into another. This is used to implement shared segments.
+ *
+ * on x86 this is very limited right now. must be exactly 1 segment.
+ *
+ * Note that we depend upon higher level VM locks to insure that things don't change while
+ * we are doing this. For example, VM should not be doing any pmap enters while it is nesting
+ * or do 2 nests at once.
+ */
+
+
+kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size) {
+
+ vm_map_offset_t vaddr, nvaddr;
+ pd_entry_t *pde,*npde;
+ unsigned int i;
+ uint64_t num_pde;
+
+ // do validity tests
+ if (size & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
+ if(vstart & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
+ if(nstart & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
+ if((size >> 28) > 65536) return KERN_INVALID_VALUE; /* Max size we can nest is 16TB */
+ if(size == 0) {
+ panic("pmap_nest: size is invalid - %016llX\n", size);
+ }
+
+ PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
+ (int) grand, (int) subord,
+ (int) (vstart>>32), (int) vstart, 0);
+
+ subord->pm_shared = TRUE;
+ nvaddr = (vm_map_offset_t)nstart;
+ num_pde = size >> PDESHIFT;
+
+ PMAP_LOCK(subord);
+ for (i = 0; i < num_pde; i++) {
+ npde = pmap_pde(subord, nvaddr);
+ while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
+ PMAP_UNLOCK(subord);
+ pmap_expand(subord, nvaddr); // pmap_expand handles races
+ PMAP_LOCK(subord);
+ npde = pmap_pde(subord, nvaddr);
+ }
+ nvaddr += NBPDE;
+ }
+
+ PMAP_UNLOCK(subord);
+
+ vaddr = (vm_map_offset_t)vstart;
+
+ PMAP_LOCK(grand);
+
+ for (i = 0;i < num_pde; i++) {
+ pd_entry_t tpde;
+
+ npde = pmap_pde(subord, nstart);
+ if (npde == 0)
+ panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
+ tpde = *npde;
+ nstart += NBPDE;
+ pde = pmap_pde(grand, vaddr);
+/* Legacy mode does not require expansion.
+ * DRK: consider a debug mode test to verify that no PTEs are extant within
+ * this range.
+ */
+ if ((0 == pde) && cpu_64bit) {
+ PMAP_UNLOCK(grand);
+ pmap_expand_pdpt(grand, vaddr);
+ PMAP_LOCK(grand);
+ pde = pmap_pde(grand, vaddr);
+ }
+
+ if (pde == 0)
+ panic("pmap_nest: no pde, grand %p vaddr 0x%llx", grand, vaddr);
+ vaddr += NBPDE;
+ pmap_store_pte(pde, tpde);
+ }
+
+ /* XXX FBDP: why do we need to flush here ? */
+ PMAP_UPDATE_TLBS(grand, vstart, vstart + size - 1);
+
+ PMAP_UNLOCK(grand);
+
+ PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * kern_return_t pmap_unnest(grand, vaddr)
+ *
+ * grand = the pmap that we will nest subord into
+ * vaddr = start of range in pmap to be unnested
+ *
+ * Removes a pmap from another. This is used to implement shared segments.
+ * On the current PPC processors, this is limited to segment (256MB) aligned
+ * segment sized ranges.
+ */
+
+kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
+
+ pd_entry_t *pde;
+ unsigned int i;
+ unsigned int num_pde;
+ addr64_t vstart, vend;
+
+ PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
+ (int) grand,
+ (int) (vaddr>>32), (int) vaddr, 0, 0);
+
+ if ((size & (pmap_nesting_size_min-1)) ||
+ (vaddr & (pmap_nesting_size_min-1))) {
+ panic("pmap_unnest(%p,0x%llx,0x%llx): unaligned...\n",
+ grand, vaddr, size);
+ }
+
+ /* align everything to PDE boundaries */
+ vstart = vaddr & ~(NBPDE-1);
+ vend = (vaddr + size + NBPDE - 1) & ~(NBPDE-1);
+ size = vend - vstart;
+
+ PMAP_LOCK(grand);
+
+ // invalidate all pdes for segment at vaddr in pmap grand
+
+ num_pde = size >> PDESHIFT;
+
+ vaddr = vstart;
+ for (i=0;i<num_pde;i++,pde++) {
+ pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
+ if (pde == 0) panic("pmap_unnest: no pde, grand %p vaddr 0x%llx\n", grand, vaddr);
+ pmap_store_pte(pde, (pd_entry_t)0);
+ vaddr += NBPDE;
+ }
+ PMAP_UPDATE_TLBS(grand, vstart, vend);
+
+ PMAP_UNLOCK(grand);
+
+ PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ return KERN_SUCCESS;
+}
+
+void
+pmap_switch(pmap_t tpmap)
+{
+ spl_t s;
+ int my_cpu;
+
+ s = splhigh(); /* Make sure interruptions are disabled */
+ my_cpu = cpu_number();
+
+ set_dirbase(tpmap, my_cpu);
+
+ splx(s);
+}
+
+
+/*
+ * disable no-execute capability on
+ * the specified pmap
+ */
+void pmap_disable_NX(pmap_t pmap) {
+
+ pmap->nx_enabled = 0;
+}
+
+void
+pt_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
+ vm_size_t *alloc_size, int *collectable, int *exhaustable)
+{
+ *count = inuse_ptepages_count;
+ *cur_size = PAGE_SIZE * inuse_ptepages_count;
+ *max_size = PAGE_SIZE * (inuse_ptepages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count);
+ *elem_size = PAGE_SIZE;
+ *alloc_size = PAGE_SIZE;
+
+ *collectable = 1;
+ *exhaustable = 0;
+}
+
+vm_offset_t pmap_cpu_high_map_vaddr(int cpu, enum high_cpu_types e)
+{
+ enum high_fixed_addresses a;
+ a = e + HIGH_CPU_END * cpu;
+ return pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+}
+
+vm_offset_t pmap_high_map_vaddr(enum high_cpu_types e)
+{
+ return pmap_cpu_high_map_vaddr(cpu_number(), e);
+}
+
+vm_offset_t pmap_high_map(pt_entry_t pte, enum high_cpu_types e)
+{
+ enum high_fixed_addresses a;
+ vm_offset_t vaddr;
+
+ a = e + HIGH_CPU_END * cpu_number();
+ vaddr = (vm_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+ pmap_store_pte(pte_unique_base + a, pte);
+
+ /* TLB flush for this page for this cpu */
+ invlpg((uintptr_t)vaddr);
+
+ return vaddr;
+}
+
+static inline void
+pmap_cpuset_NMIPI(cpu_set cpu_mask) {
+ unsigned int cpu, cpu_bit;
+ uint64_t deadline;
+
+ for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+ if (cpu_mask & cpu_bit)
+ cpu_NMI_interrupt(cpu);
+ }
+ deadline = mach_absolute_time() + (LockTimeOut >> 2);
+ while (mach_absolute_time() < deadline)
+ cpu_pause();
+}
+
+
+/*
+ * Called with pmap locked, we:
+ * - scan through per-cpu data to see which other cpus need to flush
+ * - send an IPI to each non-idle cpu to be flushed
+ * - wait for all to signal back that they are inactive or we see that
+ * they are in an interrupt handler or at a safe point
+ * - flush the local tlb is active for this pmap
+ * - return ... the caller will unlock the pmap
+ */
+void
+pmap_flush_tlbs(pmap_t pmap)
+{
+ unsigned int cpu;
+ unsigned int cpu_bit;
+ cpu_set cpus_to_signal;
+ unsigned int my_cpu = cpu_number();
+ pmap_paddr_t pmap_cr3 = pmap->pm_cr3;
+ boolean_t flush_self = FALSE;
+ uint64_t deadline;
+
+ assert((processor_avail_count < 2) ||
+ (ml_get_interrupts_enabled() && get_preemption_level() != 0));
+
+ /*
+ * Scan other cpus for matching active or task CR3.
+ * For idle cpus (with no active map) we mark them invalid but
+ * don't signal -- they'll check as they go busy.
+ * Note: for the kernel pmap we look for 64-bit shared address maps.
+ */
+ cpus_to_signal = 0;
+ for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+ if (!cpu_datap(cpu)->cpu_running)
+ continue;
+ if ((cpu_datap(cpu)->cpu_task_cr3 == pmap_cr3) ||
+ (CPU_GET_ACTIVE_CR3(cpu) == pmap_cr3) ||
+ (pmap->pm_shared) ||
+ ((pmap == kernel_pmap) &&
+ (!CPU_CR3_IS_ACTIVE(cpu) ||
+ cpu_datap(cpu)->cpu_task_map == TASK_MAP_64BIT_SHARED))) {
+ if (cpu == my_cpu) {
+ flush_self = TRUE;
+ continue;
+ }
+ cpu_datap(cpu)->cpu_tlb_invalid = TRUE;
+ __asm__ volatile("mfence");
+
+ if (CPU_CR3_IS_ACTIVE(cpu)) {
+ cpus_to_signal |= cpu_bit;
+ i386_signal_cpu(cpu, MP_TLB_FLUSH, ASYNC);
+ }
+ }
+ }
projects / apple / xnu.git / blobdiff