+/*
+* TLB Coherence Code (TLB "shootdown" code)
+*
+* Threads that belong to the same task share the same address space and
+* hence share a pmap. However, they may run on distinct cpus and thus
+* have distinct TLBs that cache page table entries. In order to guarantee
+* the TLBs are consistent, whenever a pmap is changed, all threads that
+* are active in that pmap must have their TLB updated. To keep track of
+* this information, the set of cpus that are currently using a pmap is
+* maintained within each pmap structure (cpus_using). Pmap_activate() and
+* pmap_deactivate add and remove, respectively, a cpu from this set.
+* Since the TLBs are not addressable over the bus, each processor must
+* flush its own TLB; a processor that needs to invalidate another TLB
+* needs to interrupt the processor that owns that TLB to signal the
+* update.
+*
+* Whenever a pmap is updated, the lock on that pmap is locked, and all
+* cpus using the pmap are signaled to invalidate. All threads that need
+* to activate a pmap must wait for the lock to clear to await any updates
+* in progress before using the pmap. They must ACQUIRE the lock to add
+* their cpu to the cpus_using set. An implicit assumption made
+* throughout the TLB code is that all kernel code that runs at or higher
+* than splvm blocks out update interrupts, and that such code does not
+* touch pageable pages.
+*
+* A shootdown interrupt serves another function besides signaling a
+* processor to invalidate. The interrupt routine (pmap_update_interrupt)
+* waits for the both the pmap lock (and the kernel pmap lock) to clear,
+* preventing user code from making implicit pmap updates while the
+* sending processor is performing its update. (This could happen via a
+* user data write reference that turns on the modify bit in the page
+* table). It must wait for any kernel updates that may have started
+* concurrently with a user pmap update because the IPC code
+* changes mappings.
+* Spinning on the VALUES of the locks is sufficient (rather than
+* having to acquire the locks) because any updates that occur subsequent
+* to finding the lock unlocked will be signaled via another interrupt.
+* (This assumes the interrupt is cleared before the low level interrupt code
+* calls pmap_update_interrupt()).
+*
+* The signaling processor must wait for any implicit updates in progress
+* to terminate before continuing with its update. Thus it must wait for an
+* acknowledgement of the interrupt from each processor for which such
+* references could be made. For maintaining this information, a set
+* cpus_active is used. A cpu is in this set if and only if it can
+* use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
+* this set; when all such cpus are removed, it is safe to update.
+*
+* Before attempting to acquire the update lock on a pmap, a cpu (A) must
+* be at least at the priority of the interprocessor interrupt
+* (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
+* kernel update; it would spin forever in pmap_update_interrupt() trying
+* to acquire the user pmap lock it had already acquired. Furthermore A
+* must remove itself from cpus_active. Otherwise, another cpu holding
+* the lock (B) could be in the process of sending an update signal to A,
+* and thus be waiting for A to remove itself from cpus_active. If A is
+* spinning on the lock at priority this will never happen and a deadlock
+* will result.
+*/
+
+/*
+ * Signal another CPU that it must flush its TLB
+ */
+void
+signal_cpus(
+ cpu_set use_list,
+ pmap_t pmap,
+ vm_offset_t start_addr,
+ vm_offset_t end_addr)
+{
+ register int which_cpu, j;
+ register pmap_update_list_t update_list_p;
+
+ while ((which_cpu = ffs((unsigned long)use_list)) != 0) {
+ which_cpu -= 1; /* convert to 0 origin */
+
+ update_list_p = cpu_update_list(which_cpu);
+ simple_lock(&update_list_p->lock);
+
+ j = update_list_p->count;
+ if (j >= UPDATE_LIST_SIZE) {
+ /*
+ * list overflowed. Change last item to
+ * indicate overflow.
+ */
+ update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
+ update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
+ update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS;
+ }
+ else {
+ update_list_p->item[j].pmap = pmap;
+ update_list_p->item[j].start = start_addr;
+ update_list_p->item[j].end = end_addr;
+ update_list_p->count = j+1;
+ }
+ cpu_update_needed(which_cpu) = TRUE;
+ simple_unlock(&update_list_p->lock);
+
+ /* if its the kernel pmap, ignore cpus_idle */
+ if (((cpus_idle & (1 << which_cpu)) == 0) ||
+ (pmap == kernel_pmap) || PMAP_REAL(which_cpu) == pmap)
+ {
+ i386_signal_cpu(which_cpu, MP_TLB_FLUSH, ASYNC);
+ }
+ use_list &= ~(1 << which_cpu);
+ }
+}
+
+void
+process_pmap_updates(
+ register pmap_t my_pmap)
+{
+ register int my_cpu;
+ register pmap_update_list_t update_list_p;
+ register int j;
+ register pmap_t pmap;
+
+ mp_disable_preemption();
+ my_cpu = cpu_number();
+ update_list_p = cpu_update_list(my_cpu);
+ simple_lock(&update_list_p->lock);
+
+ for (j = 0; j < update_list_p->count; j++) {
+ pmap = update_list_p->item[j].pmap;
+ if (pmap == my_pmap ||
+ pmap == kernel_pmap) {
+
+ if (pmap->ref_count <= 0) {
+ PMAP_CPU_CLR(pmap, my_cpu);
+ PMAP_REAL(my_cpu) = kernel_pmap;
+#ifdef PAE
+ set_cr3((unsigned int)kernel_pmap->pm_ppdpt);
+#else
+ set_cr3((unsigned int)kernel_pmap->pdirbase);
+#endif
+ } else
+ INVALIDATE_TLB(pmap,
+ update_list_p->item[j].start,
+ update_list_p->item[j].end);
+ }
+ }
+ update_list_p->count = 0;
+ cpu_update_needed(my_cpu) = FALSE;
+ simple_unlock(&update_list_p->lock);
+ mp_enable_preemption();
+}
+
+/*
+ * Interrupt routine for TBIA requested from other processor.
+ * This routine can also be called at all interrupts time if
+ * the cpu was idle. Some driver interrupt routines might access
+ * newly allocated vm. (This is the case for hd)
+ */
+void
+pmap_update_interrupt(void)
+{
+ register int my_cpu;
+ spl_t s;
+ register pmap_t my_pmap;
+
+ mp_disable_preemption();
+ my_cpu = cpu_number();
+
+ /*
+ * Raise spl to splvm (above splip) to block out pmap_extract
+ * from IO code (which would put this cpu back in the active
+ * set).
+ */
+ s = splhigh();
+
+ my_pmap = PMAP_REAL(my_cpu);
+
+ if (!(my_pmap && pmap_in_use(my_pmap, my_cpu)))
+ my_pmap = kernel_pmap;
+
+ do {
+ LOOP_VAR;
+
+ /*
+ * Indicate that we're not using either user or kernel
+ * pmap.
+ */
+ i_bit_clear(my_cpu, &cpus_active);
+
+ /*
+ * Wait for any pmap updates in progress, on either user
+ * or kernel pmap.
+ */
+ while (*(volatile int *)(&my_pmap->lock.interlock.lock_data) ||
+ *(volatile int *)(&kernel_pmap->lock.interlock.lock_data)) {
+ LOOP_CHECK("pmap_update_interrupt", my_pmap);
+ cpu_pause();
+ }
+
+ process_pmap_updates(my_pmap);
+
+ i_bit_set(my_cpu, &cpus_active);
+
+ } while (cpu_update_needed(my_cpu));
+
+ splx(s);
+ mp_enable_preemption();
+}
+
projects / apple / xnu.git / blobdiff