xnu-3789.1.32.tar.gz

[apple/xnu.git] / osfmk / x86_64 / pmap_pcid.c

diff --git a/osfmk/x86_64/pmap_pcid.c b/osfmk/x86_64/pmap_pcid.c
index c8fef93b42c65f3eab3345229436766c28c7991d..2a7280d41ec4d0707b1c70114a45f21f0f63bb44 100644 (file)
--- a/osfmk/x86_64/pmap_pcid.c
+++ b/osfmk/x86_64/pmap_pcid.c
@@ -242,15 +242,24 @@ void      pmap_destroy_pcid_sync(pmap_t p) {
                        pmap_pcid_deallocate_pcid(i, p);
 }
 
                        pmap_pcid_deallocate_pcid(i, p);
 }
 

-pcid_t pcid_for_pmap_cpu_tuple(pmap_t pmap, int ccpu) {
-       return pmap->pmap_pcid_cpus[ccpu];
+pcid_t pcid_for_pmap_cpu_tuple(pmap_t cpmap, thread_t cthread, int ccpu) {
+       pmap_t active_pmap = cpmap;
+
+       if (__improbable(cpmap->pagezero_accessible)) {
+               if ((cthread->machine.specFlags & CopyIOActive) == 0) {
+                       active_pmap = kernel_pmap;
+               }
+       }
+
+       return active_pmap->pmap_pcid_cpus[ccpu];

 }
 }

+

 #if PMAP_ASSERT
 #define PCID_RECORD_SIZE 128
 uint64_t pcid_record_array[PCID_RECORD_SIZE];
 #endif
 
 #if PMAP_ASSERT
 #define PCID_RECORD_SIZE 128
 uint64_t pcid_record_array[PCID_RECORD_SIZE];
 #endif
 

-void   pmap_pcid_activate(pmap_t tpmap, int ccpu) {
+void   pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t copyio) {

        pcid_t          new_pcid = tpmap->pmap_pcid_cpus[ccpu];
        pmap_t          last_pmap;
        boolean_t       pcid_conflict = FALSE, pending_flush = FALSE;
        pcid_t          new_pcid = tpmap->pmap_pcid_cpus[ccpu];
        pmap_t          last_pmap;
        boolean_t       pcid_conflict = FALSE, pending_flush = FALSE;


@@ -259,8 +268,9 @@ void        pmap_pcid_activate(pmap_t tpmap, int ccpu) {
        if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {
                new_pcid = tpmap->pmap_pcid_cpus[ccpu] = pmap_pcid_allocate_pcid(ccpu);
        }
        if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {
                new_pcid = tpmap->pmap_pcid_cpus[ccpu] = pmap_pcid_allocate_pcid(ccpu);
        }

+

        pmap_assert(new_pcid != PMAP_PCID_INVALID_PCID);
        pmap_assert(new_pcid != PMAP_PCID_INVALID_PCID);

-#ifdef PCID_ASSERT     
+#ifdef PCID_ASSERT

        cpu_datap(ccpu)->cpu_last_pcid = cpu_datap(ccpu)->cpu_active_pcid;
 #endif
        cpu_datap(ccpu)->cpu_active_pcid = new_pcid;
        cpu_datap(ccpu)->cpu_last_pcid = cpu_datap(ccpu)->cpu_active_pcid;
 #endif
        cpu_datap(ccpu)->cpu_active_pcid = new_pcid;


@@ -268,7 +278,7 @@ void        pmap_pcid_activate(pmap_t tpmap, int ccpu) {
        pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
        if (__probable(pending_flush == FALSE)) {
                last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];
        pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
        if (__probable(pending_flush == FALSE)) {
                last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];

-               pcid_conflict = ((last_pmap != NULL) &&(tpmap != last_pmap));
+               pcid_conflict = ((last_pmap != NULL) && (tpmap != last_pmap));

        }
        if (__improbable(pending_flush || pcid_conflict)) {
                pmap_pcid_validate_cpu(tpmap, ccpu);
        }
        if (__improbable(pending_flush || pcid_conflict)) {
                pmap_pcid_validate_cpu(tpmap, ccpu);


@@ -277,7 +287,8 @@ void        pmap_pcid_activate(pmap_t tpmap, int ccpu) {
        cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
 
        pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);
        cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
 
        pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);

-       pmap_assert(((tpmap ==  kernel_pmap) && new_pcid == 0) || ((new_pcid != PMAP_PCID_INVALID_PCID) && (new_pcid != 0)));
+       pmap_assert(((tpmap ==  kernel_pmap) && new_pcid == 0) ||
+           ((new_pcid != PMAP_PCID_INVALID_PCID) && (new_pcid != 0)));

 #if    PMAP_ASSERT
        pcid_record_array[ccpu % PCID_RECORD_SIZE] = tpmap->pm_cr3 | new_pcid | (((uint64_t)(!(pending_flush || pcid_conflict))) <<63);
        pml4_entry_t *pml4 = pmap64_pml4(tpmap, 0ULL);
 #if    PMAP_ASSERT
        pcid_record_array[ccpu % PCID_RECORD_SIZE] = tpmap->pm_cr3 | new_pcid | (((uint64_t)(!(pending_flush || pcid_conflict))) <<63);
        pml4_entry_t *pml4 = pmap64_pml4(tpmap, 0ULL);


@@ -285,7 +296,19 @@ void       pmap_pcid_activate(pmap_t tpmap, int ccpu) {
        if (pml4[KERNEL_PML4_INDEX] != kernel_pmap->pm_pml4[KERNEL_PML4_INDEX])
                __asm__ volatile("int3");
 #endif /* PMAP_ASSERT */
        if (pml4[KERNEL_PML4_INDEX] != kernel_pmap->pm_pml4[KERNEL_PML4_INDEX])
                __asm__ volatile("int3");
 #endif /* PMAP_ASSERT */

-       set_cr3_composed(tpmap->pm_cr3, new_pcid, !(pending_flush || pcid_conflict));
+
+       pmap_paddr_t ncr3 = tpmap->pm_cr3;
+
+       if (__improbable(nopagezero)) {
+               pending_flush = TRUE;
+               if (copyio == FALSE) {
+                       new_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];
+                       ncr3 = kernel_pmap->pm_cr3;
+               }
+               cpu_datap(ccpu)->cpu_kernel_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];
+       }
+
+       set_cr3_composed(ncr3, new_pcid, !(pending_flush || pcid_conflict));

 
        if (!pending_flush) {
                /* We did not previously observe a pending invalidation for this
 
        if (!pending_flush) {
                /* We did not previously observe a pending invalidation for this


@@ -300,7 +323,7 @@ void        pmap_pcid_activate(pmap_t tpmap, int ccpu) {
                pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
                if (__improbable(pending_flush != 0)) {
                        pmap_pcid_validate_cpu(tpmap, ccpu);
                pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
                if (__improbable(pending_flush != 0)) {
                        pmap_pcid_validate_cpu(tpmap, ccpu);

-                       set_cr3_composed(tpmap->pm_cr3, new_pcid, FALSE);
+                       set_cr3_composed(ncr3, new_pcid, FALSE);

                }
        }
        cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = &(tpmap->pmap_pcid_coherency_vector[ccpu]);
                }
        }
        cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = &(tpmap->pmap_pcid_coherency_vector[ccpu]);