+
+
+mapwindow_t *
+pmap_get_mapwindow(pt_entry_t pentry)
+{
+ mapwindow_t *mp;
+ int i;
+ boolean_t istate;
+
+ /*
+ * can be called from hardware interrupt context
+ * so we need to protect the lookup process
+ */
+ istate = ml_set_interrupts_enabled(FALSE);
+
+ /*
+ * Note: 0th map reserved for pmap_pte()
+ */
+ for (i = PMAP_NWINDOWS_FIRSTFREE; i < PMAP_NWINDOWS; i++) {
+ mp = ¤t_cpu_datap()->cpu_pmap->mapwindow[i];
+
+ if (*mp->prv_CMAP == 0) {
+ *mp->prv_CMAP = pentry;
+ break;
+ }
+ }
+ if (i >= PMAP_NWINDOWS)
+ mp = NULL;
+ (void) ml_set_interrupts_enabled(istate);
+
+ return (mp);
+}
+
+
+/*
+ * kern_return_t pmap_nest(grand, subord, vstart, size)
+ *
+ * grand = the pmap that we will nest subord into
+ * subord = the pmap that goes into the grand
+ * vstart = start of range in pmap to be inserted
+ * nstart = start of range in pmap nested pmap
+ * size = Size of nest area (up to 16TB)
+ *
+ * Inserts a pmap into another. This is used to implement shared segments.
+ *
+ * on x86 this is very limited right now. must be exactly 1 segment.
+ *
+ * Note that we depend upon higher level VM locks to insure that things don't change while
+ * we are doing this. For example, VM should not be doing any pmap enters while it is nesting
+ * or do 2 nests at once.
+ */
+
+
+kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size) {
+
+ vm_map_offset_t vaddr, nvaddr;
+ pd_entry_t *pde,*npde;
+ unsigned int i, need_flush;
+ unsigned int num_pde;
+ spl_t s;
+
+ // do validity tests
+
+ if(size & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this for multiples of 256MB */
+ if((size >> 28) > 65536) return KERN_INVALID_VALUE; /* Max size we can nest is 16TB */
+ if(vstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this aligned to 256MB */
+ if(nstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this aligned to 256MB */
+ if(size == 0) {
+ panic("pmap_nest: size is invalid - %016llX\n", size);
+ }
+ if ((size >> 28) != 1) panic("pmap_nest: size 0x%llx must be 0x%x", size, NBPDE);
+
+ subord->pm_shared = TRUE;
+
+ // prepopulate subord pmap pde's if necessary
+
+ if (cpu_64bit) {
+ s = splhigh();
+ while (PD_ENTRY_NULL == (npde = pmap_pde(subord, nstart))) {
+ splx(s);
+ pmap_expand(subord, nstart);
+ s = splhigh();
+ }
+ splx(s);
+ }
+
+ PMAP_READ_LOCK(subord,s);
+ nvaddr = (vm_map_offset_t)nstart;
+ need_flush = 0;
+ num_pde = size >> PDESHIFT;
+
+ for (i=0;i<num_pde;i++) {
+ npde = pmap_pde(subord, nvaddr);
+ if ((0 == npde) || (*npde++ & INTEL_PTE_VALID) == 0) {
+ PMAP_READ_UNLOCK(subord,s);
+ pmap_expand(subord, nvaddr); // pmap_expand handles races
+ PMAP_READ_LOCK(subord,s);
+ need_flush++;
+ }
+ nvaddr += NBPDE;
+ }
+
+ if (need_flush) {
+ nvaddr = (vm_map_offset_t)nstart;
+ PMAP_UPDATE_TLBS(subord, nvaddr, nvaddr + (1 << 28) -1 );
+ }
+ PMAP_READ_UNLOCK(subord,s);
+
+ // copy pde's from subord pmap into grand pmap
+
+ if (cpu_64bit) {
+ s = splhigh();
+ while (PD_ENTRY_NULL == (pde = pmap_pde(grand, vstart))) {
+ splx(s);
+ pmap_expand(grand, vstart);
+ s = splhigh();
+ }
+ splx(s);
+ }
+
+ PMAP_READ_LOCK(grand,s);
+ vaddr = (vm_map_offset_t)vstart;
+ for (i=0;i<num_pde;i++,pde++) {
+ pd_entry_t tpde;
+ npde = pmap_pde(subord, nstart);
+ if (npde == 0) panic("pmap_nest: no npde, subord 0x%x nstart 0x%llx", subord, nstart);
+ tpde = *npde;
+ nstart += NBPDE;
+ pde = pmap_pde(grand, vaddr);
+ if (pde == 0) panic("pmap_nest: no pde, grand 0x%x vaddr 0x%llx", grand, vaddr);
+ vaddr += NBPDE;
+ pmap_store_pte(pde, tpde);
+ }
+ PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1 << 28) -1 );
+
+ PMAP_READ_UNLOCK(grand,s);
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * kern_return_t pmap_unnest(grand, vaddr)
+ *
+ * grand = the pmap that we will nest subord into
+ * vaddr = start of range in pmap to be unnested
+ *
+ * Removes a pmap from another. This is used to implement shared segments.
+ * On the current PPC processors, this is limited to segment (256MB) aligned
+ * segment sized ranges.
+ */
+
+kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr) {
+
+ spl_t s;
+ pd_entry_t *pde;
+ unsigned int i;
+ unsigned int num_pde;
+
+ PMAP_READ_LOCK(grand,s);
+
+ // invalidate all pdes for segment at vaddr in pmap grand
+
+ num_pde = (1<<28) >> PDESHIFT;
+
+ for (i=0;i<num_pde;i++,pde++) {
+ pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
+ if (pde == 0) panic("pmap_unnest: no pde, grand 0x%x vaddr 0x%llx\n", grand, vaddr);
+ pmap_store_pte(pde, (pd_entry_t)0);
+ vaddr += NBPDE;
+ }
+ PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1<<28) -1 );
+
+ PMAP_READ_UNLOCK(grand,s);
+
+ return KERN_SUCCESS; /* Bye, bye, butterfly... */
+}
+
+void
+pmap_switch(pmap_t tpmap)
+{
+ spl_t s;
+ int my_cpu;
+
+ s = splhigh(); /* Make sure interruptions are disabled */
+ my_cpu = cpu_number();
+
+ set_dirbase(tpmap, my_cpu);
+
+ splx(s);
+}
+
+
+/*
+ * disable no-execute capability on
+ * the specified pmap
+ */
+void pmap_disable_NX(pmap_t pmap) {
+
+ pmap->nx_enabled = 0;
+}
+
+void
+pt_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
+ vm_size_t *alloc_size, int *collectable, int *exhaustable)
+{
+ *count = inuse_ptepages_count;
+ *cur_size = PAGE_SIZE * inuse_ptepages_count;
+ *max_size = PAGE_SIZE * (inuse_ptepages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count);
+ *elem_size = PAGE_SIZE;
+ *alloc_size = PAGE_SIZE;
+
+ *collectable = 1;
+ *exhaustable = 0;
+}
+
+vm_offset_t pmap_cpu_high_map_vaddr(int cpu, enum high_cpu_types e)
+{
+ enum high_fixed_addresses a;
+ a = e + HIGH_CPU_END * cpu;
+ return pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+}
+
+vm_offset_t pmap_high_map_vaddr(enum high_cpu_types e)
+{
+ return pmap_cpu_high_map_vaddr(cpu_number(), e);
+}
+
+vm_offset_t pmap_high_map(pt_entry_t pte, enum high_cpu_types e)
+{
+ enum high_fixed_addresses a;
+ vm_offset_t vaddr;
+
+ a = e + HIGH_CPU_END * cpu_number();
+ vaddr = (vm_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+ *(pte_unique_base + a) = pte;
+
+ /* TLB flush for this page for this cpu */
+ invlpg((uintptr_t)vaddr);
+
+ return vaddr;
+}
+
+
+/*
+ * Called with pmap locked, we:
+ * - scan through per-cpu data to see which other cpus need to flush
+ * - send an IPI to each non-idle cpu to be flushed
+ * - wait for all to signal back that they are inactive or we see that
+ * they are in an interrupt handler or at a safe point
+ * - flush the local tlb is active for this pmap
+ * - return ... the caller will unlock the pmap
+ */
+void
+pmap_flush_tlbs(pmap_t pmap)
+{
+ unsigned int cpu;
+ unsigned int cpu_bit;
+ cpu_set cpus_to_signal;
+ unsigned int my_cpu = cpu_number();
+ pmap_paddr_t pmap_cr3 = pmap->pm_cr3;
+ boolean_t flush_self = FALSE;
+ uint64_t deadline;
+
+ assert(!ml_get_interrupts_enabled());
+
+ /*
+ * Scan other cpus for matching active or task CR3.
+ * For idle cpus (with no active map) we mark them invalid but
+ * don't signal -- they'll check as they go busy.
+ * Note: for the kernel pmap we look for 64-bit shared address maps.
+ */
+ cpus_to_signal = 0;
+ for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+ if (!cpu_datap(cpu)->cpu_running)
+ continue;
+ if ((cpu_datap(cpu)->cpu_task_cr3 == pmap_cr3) ||
+ (CPU_GET_ACTIVE_CR3(cpu) == pmap_cr3) ||
+ (pmap->pm_shared) ||
+ ((pmap == kernel_pmap) &&
+ (!CPU_CR3_IS_ACTIVE(cpu) ||
+ cpu_datap(cpu)->cpu_task_map == TASK_MAP_64BIT_SHARED))) {
+ if (cpu == my_cpu) {
+ flush_self = TRUE;
+ continue;
+ }
+ cpu_datap(cpu)->cpu_tlb_invalid = TRUE;
+ __asm__ volatile("mfence");
+
+ if (CPU_CR3_IS_ACTIVE(cpu)) {
+ cpus_to_signal |= cpu_bit;
+ i386_signal_cpu(cpu, MP_TLB_FLUSH, ASYNC);
+ }
+ }
+ }
+
+ if (cpus_to_signal) {
+ KERNEL_DEBUG(0xef800024 | DBG_FUNC_START, cpus_to_signal, 0, 0, 0, 0);
+
+ deadline = mach_absolute_time() + LockTimeOut;
+ /*
+ * Wait for those other cpus to acknowledge
+ */
+ for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+ while ((cpus_to_signal & cpu_bit) != 0) {
+ if (!cpu_datap(cpu)->cpu_running ||
+ cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
+ !CPU_CR3_IS_ACTIVE(cpu)) {
+ cpus_to_signal &= ~cpu_bit;
+ break;
+ }
+ if (mach_absolute_time() > deadline)
+ panic("pmap_flush_tlbs() "
+ "timeout pmap=%p cpus_to_signal=%p",
+ pmap, cpus_to_signal);
+ cpu_pause();
+ }
+ if (cpus_to_signal == 0)
+ break;
+ }
+ KERNEL_DEBUG(0xef800024 | DBG_FUNC_END, cpus_to_signal, 0, 0, 0, 0);
+ }
+
+ /*
+ * Flush local tlb if required.
+ * We need this flush even if the pmap being changed
+ * is the user map... in case we do a copyin/out
+ * before returning to user mode.
+ */
+ if (flush_self)
+ flush_tlb();
+
+}
+
+void
+process_pmap_updates(void)
+{
+ flush_tlb();
+
+ current_cpu_datap()->cpu_tlb_invalid = FALSE;
+ __asm__ volatile("mfence");
+}
+
+void
+pmap_update_interrupt(void)
+{
+ KERNEL_DEBUG(0xef800028 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ assert(!ml_get_interrupts_enabled());
+
+ process_pmap_updates();
+
+ KERNEL_DEBUG(0xef800028 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+}
+
+
+unsigned int pmap_cache_attributes(ppnum_t pn) {
+
+ if (!pmap_valid_page(pn))
+ return (VM_WIMG_IO);
+
+ return (VM_WIMG_COPYBACK);
+}
+
+#ifdef PMAP_DEBUG
+void
+pmap_dump(pmap_t p)
+{
+ int i;
+
+ kprintf("pmap 0x%x\n",p);
+
+ kprintf(" pm_cr3 0x%llx\n",p->pm_cr3);
+ kprintf(" pm_pml4 0x%x\n",p->pm_pml4);
+ kprintf(" pm_pdpt 0x%x\n",p->pm_pdpt);
+
+ kprintf(" pml4[0] 0x%llx\n",*p->pm_pml4);
+ for (i=0;i<8;i++)
+ kprintf(" pdpt[%d] 0x%llx\n",i, p->pm_pdpt[i]);
+}
+
+void pmap_dump_wrap(void)
+{
+ pmap_dump(current_cpu_datap()->cpu_active_thread->task->map->pmap);
+}
+
+void
+dump_4GB_pdpt(pmap_t p)
+{
+ int spl;
+ pdpt_entry_t *user_pdptp;
+ pdpt_entry_t *kern_pdptp;
+ pdpt_entry_t *pml4p;
+
+ spl = splhigh();
+ while ((user_pdptp = pmap64_pdpt(p, 0x0)) == PDPT_ENTRY_NULL) {
+ splx(spl);
+ pmap_expand_pml4(p, 0x0);
+ spl = splhigh();
+ }
+ kern_pdptp = kernel_pmap->pm_pdpt;
+ if (kern_pdptp == NULL)
+ panic("kern_pdptp == NULL");
+ kprintf("dump_4GB_pdpt(%p)\n"
+ "kern_pdptp=%p (phys=0x%016llx)\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "user_pdptp=%p (phys=0x%016llx)\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n",
+ p, kern_pdptp, kvtophys(kern_pdptp),
+ kern_pdptp+0, *(kern_pdptp+0),
+ kern_pdptp+1, *(kern_pdptp+1),
+ kern_pdptp+2, *(kern_pdptp+2),
+ kern_pdptp+3, *(kern_pdptp+3),
+ kern_pdptp+4, *(kern_pdptp+4),
+ user_pdptp, kvtophys(user_pdptp),
+ user_pdptp+0, *(user_pdptp+0),
+ user_pdptp+1, *(user_pdptp+1),
+ user_pdptp+2, *(user_pdptp+2),
+ user_pdptp+3, *(user_pdptp+3),
+ user_pdptp+4, *(user_pdptp+4));
+ kprintf("user pm_cr3=0x%016llx pm_hold=0x%08x pm_pml4=0x%08x\n",
+ p->pm_cr3, p->pm_hold, p->pm_pml4);
+ pml4p = (pdpt_entry_t *)p->pm_hold;
+ if (pml4p == NULL)
+ panic("user pml4p == NULL");
+ kprintf("\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n",
+ pml4p+0, *(pml4p),
+ pml4p+KERNEL_UBER_PML4_INDEX, *(pml4p+KERNEL_UBER_PML4_INDEX));
+ kprintf("kern pm_cr3=0x%016llx pm_hold=0x%08x pm_pml4=0x%08x\n",
+ kernel_pmap->pm_cr3, kernel_pmap->pm_hold, kernel_pmap->pm_pml4);
+ pml4p = (pdpt_entry_t *)kernel_pmap->pm_hold;
+ if (pml4p == NULL)
+ panic("kern pml4p == NULL");
+ kprintf("\t 0x%08x: 0x%016llx\n"
+ "\t 0x%08x: 0x%016llx\n",
+ pml4p+0, *(pml4p),
+ pml4p+511, *(pml4p+511));
+ splx(spl);
+}
+
+void dump_4GB_pdpt_thread(thread_t tp)
+{
+ dump_4GB_pdpt(tp->map->pmap);
+}
+
+
+#endif
projects / apple / xnu.git / blobdiff