#include <vm/vm_map.h>
#include <i386/pmap_internal.h>
#include <i386/pmap_pcid.h>
-#include <mach/branch_predicates.h>
/*
* PCID (Process context identifier) aka tagged TLB support.
uint32_t pmap_pcid_ncpus;
boolean_t pmap_pcid_disabled = FALSE;
+pcid_cdata_t pcid_data[MAX_CPUS] __attribute__((aligned(64)));
void pmap_pcid_configure(void) {
int ccpu = cpu_number();
kprintf("PMAP: PCID feature disabled %u\n", pmap_pcid_disabled);
}
/* no_shared_cr3+PCID is currently unsupported */
+ //todo remove nscr3
#if DEBUG
if (pmap_pcid_disabled == FALSE)
no_shared_cr3 = FALSE;
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp =
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel =
&(kernel_pmap->pmap_pcid_coherency_vector[ccpu]);
- cpu_datap(ccpu)->cpu_pcid_refcounts[0] = 1;
+ cpu_datap(ccpu)->cpu_pcid_data = &pcid_data[ccpu];
+ cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0] = 1;
}
}
int i;
pcid_ref_t cur_min = 0xFF;
uint32_t cur_min_index = ~1;
- pcid_ref_t *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_refcounts[0];
+ pcid_ref_t *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0];
pcid_ref_t old_count;
- if ((i = cpu_datap(ccpu)->cpu_pcid_free_hint) != 0) {
+ if ((i = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint) != 0) {
if (cpu_pcid_refcounts[i] == 0) {
(void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
- cpu_datap(ccpu)->cpu_pcid_free_hint = 0;
+ cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = 0;
return i;
}
}
if (cur_refcount == 0) {
(void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
return i;
- }
- else {
+ } else {
if (cur_refcount < cur_min) {
cur_min_index = i;
cur_min = cur_refcount;
old_count = __sync_fetch_and_add(&cpu_pcid_refcounts[cur_min_index], 1);
pmap_assert(old_count < PMAP_PCID_MAX_REFCOUNT);
- return cur_min_index;
+ return (cur_min_index);
}
void pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {
if (pcid == PMAP_PCID_INVALID_PCID)
return;
- lp = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid];
+ lp = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid];
pmap_assert(pcid > 0 && pcid < PMAP_PCID_MAX_PCID);
- pmap_assert(cpu_datap(ccpu)->cpu_pcid_refcounts[pcid] >= 1);
+ pmap_assert(cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid] >= 1);
if (lp == tpmap)
- (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);
+ (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);
- if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_refcounts[pcid], 1)) == 1) {
- cpu_datap(ccpu)->cpu_pcid_free_hint = pcid;
+ if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid], 1)) == 1) {
+ cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = pcid;
}
pmap_assert(prior_count <= PMAP_PCID_MAX_REFCOUNT);
}
pmap_pcid_deallocate_pcid(i, p);
}
-pcid_t pcid_for_pmap_cpu_tuple(pmap_t pmap, int ccpu) {
- return pmap->pmap_pcid_cpus[ccpu];
+pcid_t pcid_for_pmap_cpu_tuple(pmap_t cpmap, thread_t cthread, int ccpu) {
+ pmap_t active_pmap = cpmap;
+
+ if (__improbable(cpmap->pagezero_accessible)) {
+ if ((cthread->machine.specFlags & CopyIOActive) == 0) {
+ active_pmap = kernel_pmap;
+ }
+ }
+
+ return active_pmap->pmap_pcid_cpus[ccpu];
}
+int npz = 0;
+
#if PMAP_ASSERT
#define PCID_RECORD_SIZE 128
uint64_t pcid_record_array[PCID_RECORD_SIZE];
#endif
-void pmap_pcid_activate(pmap_t tpmap, int ccpu) {
+void pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t copyio) {
pcid_t new_pcid = tpmap->pmap_pcid_cpus[ccpu];
pmap_t last_pmap;
boolean_t pcid_conflict = FALSE, pending_flush = FALSE;
+ pcid_cdata_t *pcdata = cpu_datap(ccpu)->cpu_pcid_data;
pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled);
if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {
new_pcid = tpmap->pmap_pcid_cpus[ccpu] = pmap_pcid_allocate_pcid(ccpu);
}
+
pmap_assert(new_pcid != PMAP_PCID_INVALID_PCID);
-#ifdef PCID_ASSERT
+#ifdef PCID_ASSERT
cpu_datap(ccpu)->cpu_last_pcid = cpu_datap(ccpu)->cpu_active_pcid;
#endif
cpu_datap(ccpu)->cpu_active_pcid = new_pcid;
pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
if (__probable(pending_flush == FALSE)) {
- last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];
- pcid_conflict = ((last_pmap != NULL) &&(tpmap != last_pmap));
+ last_pmap = pcdata->cpu_pcid_last_pmap_dispatched[new_pcid];
+ pcid_conflict = ((last_pmap != NULL) && (tpmap != last_pmap));
}
if (__improbable(pending_flush || pcid_conflict)) {
pmap_pcid_validate_cpu(tpmap, ccpu);
}
/* Consider making this a unique id */
- cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
+ pcdata->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);
- pmap_assert(((tpmap == kernel_pmap) && new_pcid == 0) || ((new_pcid != PMAP_PCID_INVALID_PCID) && (new_pcid != 0)));
+ pmap_assert(((tpmap == kernel_pmap) && new_pcid == 0) ||
+ ((new_pcid != PMAP_PCID_INVALID_PCID) && (new_pcid != 0)));
#if PMAP_ASSERT
pcid_record_array[ccpu % PCID_RECORD_SIZE] = tpmap->pm_cr3 | new_pcid | (((uint64_t)(!(pending_flush || pcid_conflict))) <<63);
pml4_entry_t *pml4 = pmap64_pml4(tpmap, 0ULL);
if (pml4[KERNEL_PML4_INDEX] != kernel_pmap->pm_pml4[KERNEL_PML4_INDEX])
__asm__ volatile("int3");
#endif /* PMAP_ASSERT */
- set_cr3_composed(tpmap->pm_cr3, new_pcid, !(pending_flush || pcid_conflict));
+
+ pmap_paddr_t ncr3 = tpmap->pm_cr3;
+
+ if (__improbable(nopagezero)) {
+ pending_flush = TRUE;
+ if (copyio == FALSE) {
+ new_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];
+ ncr3 = kernel_pmap->pm_cr3;
+ }
+ cpu_datap(ccpu)->cpu_kernel_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];
+ npz++;
+ }
+
+ uint64_t preserve = !(pending_flush || pcid_conflict);
+ set_cr3_composed(ncr3, new_pcid, preserve);
+#if DEBUG
+ cpu_datap(ccpu)->cpu_pcid_last_cr3 = ncr3 | new_pcid | preserve << 63;
+#endif
+ uint64_t spcid = (new_pcid + PMAP_PCID_MAX_PCID);
+ if (new_pcid == 0) {
+ spcid = 0;
+ }
+ uint64_t scr3 = tpmap->pm_ucr3 | spcid;
+
+ cpu_datap(ccpu)->cpu_ucr3 = scr3;
+ cpu_shadowp(ccpu)->cpu_ucr3 = scr3;
+
+ cpu_shadowp(ccpu)->cpu_task_cr3 = ncr3 | new_pcid;
if (!pending_flush) {
/* We did not previously observe a pending invalidation for this
pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
if (__improbable(pending_flush != 0)) {
pmap_pcid_validate_cpu(tpmap, ccpu);
- set_cr3_composed(tpmap->pm_cr3, new_pcid, FALSE);
+ set_cr3_composed(ncr3, new_pcid, FALSE);
}
}
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = &(tpmap->pmap_pcid_coherency_vector[ccpu]);
projects / apple / xnu.git / blobdiff