xnu-4903.221.2.tar.gz

[apple/xnu.git] / osfmk / x86_64 / pmap.c

diff --git a/osfmk/x86_64/pmap.c b/osfmk/x86_64/pmap.c
index cb643816851259087aac2b0a9c89cd9b980b10f1..8be1ce0deafbf2d15c95094038431cf9665272ac 100644 (file)
--- a/osfmk/x86_64/pmap.c
+++ b/osfmk/x86_64/pmap.c
@@ -138,6 +138,7 @@
 #endif
 
 #include <vm/vm_protos.h>
 #endif
 
 #include <vm/vm_protos.h>

+#include <san/kasan.h>

 
 #include <i386/mp.h>
 #include <i386/mp_desc.h>
 
 #include <i386/mp.h>
 #include <i386/mp_desc.h>


@@ -238,6 +239,7 @@ pmap_t              kernel_pmap;
 struct zone    *pmap_zone;             /* zone of pmap structures */
 
 struct zone    *pmap_anchor_zone;
 struct zone    *pmap_zone;             /* zone of pmap structures */
 
 struct zone    *pmap_anchor_zone;

+struct zone    *pmap_uanchor_zone;

 int            pmap_debug = 0;         /* flag for debugging prints */
 
 unsigned int   inuse_ptepages_count = 0;
 int            pmap_debug = 0;         /* flag for debugging prints */
 
 unsigned int   inuse_ptepages_count = 0;


@@ -282,12 +284,21 @@ pmap_map(
        vm_prot_t       prot,
        unsigned int    flags)
 {
        vm_prot_t       prot,
        unsigned int    flags)
 {

+       kern_return_t   kr;

        int             ps;
 
        ps = PAGE_SIZE;
        while (start_addr < end_addr) {
        int             ps;
 
        ps = PAGE_SIZE;
        while (start_addr < end_addr) {

-               pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
-                          (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+               kr = pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
+                               (ppnum_t) i386_btop(start_addr), prot, VM_PROT_NONE, flags, TRUE);
+
+               if (kr != KERN_SUCCESS) {
+                       panic("%s: failed pmap_enter, "
+                             "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
+                             __FUNCTION__,
+                             (void *)virt, (void *)start_addr, (void *)end_addr, prot, flags);
+               }
+

                virt += ps;
                start_addr += ps;
        }
                virt += ps;
                start_addr += ps;
        }


@@ -313,20 +324,18 @@ void
 pmap_cpu_init(void)
 {
        cpu_data_t      *cdp = current_cpu_datap();
 pmap_cpu_init(void)
 {
        cpu_data_t      *cdp = current_cpu_datap();

-       /*
-        * Here early in the life of a processor (from cpu_mode_init()).
-        * Ensure global page feature is disabled at this point.
-        */

 
 

-       set_cr4(get_cr4() &~ CR4_PGE);
+       set_cr4(get_cr4() | CR4_PGE);

 
        /*
         * Initialize the per-cpu, TLB-related fields.
         */
        cdp->cpu_kernel_cr3 = kernel_pmap->pm_cr3;
 
        /*
         * Initialize the per-cpu, TLB-related fields.
         */
        cdp->cpu_kernel_cr3 = kernel_pmap->pm_cr3;

+       cpu_shadowp(cdp->cpu_number)->cpu_kernel_cr3 = cdp->cpu_kernel_cr3;

        cdp->cpu_active_cr3 = kernel_pmap->pm_cr3;
        cdp->cpu_tlb_invalid = FALSE;
        cdp->cpu_task_map = TASK_MAP_64BIT;
        cdp->cpu_active_cr3 = kernel_pmap->pm_cr3;
        cdp->cpu_tlb_invalid = FALSE;
        cdp->cpu_task_map = TASK_MAP_64BIT;

+

        pmap_pcid_configure();
        if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
                pmap_smep_enabled = TRUE;
        pmap_pcid_configure();
        if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
                pmap_smep_enabled = TRUE;


@@ -354,10 +363,12 @@ pmap_cpu_init(void)
                }
        }
 
                }
        }
 

+#if !MONOTONIC

        if (cdp->cpu_fixed_pmcs_enabled) {
                boolean_t enable = TRUE;
                cpu_pmc_control(&enable);
        }
        if (cdp->cpu_fixed_pmcs_enabled) {
                boolean_t enable = TRUE;
                cpu_pmc_control(&enable);
        }

+#endif /* !MONOTONIC */

 }
 
 static uint32_t pmap_scale_shift(void) {
 }
 
 static uint32_t pmap_scale_shift(void) {


@@ -403,16 +414,15 @@ pmap_bootstrap(
        kernel_pmap->nx_enabled = TRUE;
        kernel_pmap->pm_task_map = TASK_MAP_64BIT;
        kernel_pmap->pm_obj = (vm_object_t) NULL;
        kernel_pmap->nx_enabled = TRUE;
        kernel_pmap->pm_task_map = TASK_MAP_64BIT;
        kernel_pmap->pm_obj = (vm_object_t) NULL;

-       kernel_pmap->dirbase = (pd_entry_t *)((uintptr_t)IdlePTD);
-       kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT);

        kernel_pmap->pm_pml4 = IdlePML4;
        kernel_pmap->pm_pml4 = IdlePML4;

+       kernel_pmap->pm_upml4 = IdlePML4;

        kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
        kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);

+       kernel_pmap->pm_ucr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);

        kernel_pmap->pm_eptp = 0;
        kernel_pmap->pm_eptp = 0;

-       pmap_pcid_initialize_kernel(kernel_pmap);

 
 

-       
+       pmap_pcid_initialize_kernel(kernel_pmap);

 
 

-       current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
+       current_cpu_datap()->cpu_kernel_cr3 = cpu_shadowp(cpu_number())->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;

 
        nkpt = NKPT;
        OSAddAtomic(NKPT,  &inuse_ptepages_count);
 
        nkpt = NKPT;
        OSAddAtomic(NKPT,  &inuse_ptepages_count);


@@ -489,6 +499,7 @@ pmap_bootstrap(
        printf("Stack canary: 0x%lx\n", __stack_chk_guard[0]);
        printf("early_random(): 0x%qx\n", early_random());
 #endif
        printf("Stack canary: 0x%lx\n", __stack_chk_guard[0]);
        printf("early_random(): 0x%qx\n", early_random());
 #endif

+#if    DEVELOPMENT || DEBUG

        boolean_t ptmp;
        /* Check if the user has requested disabling stack or heap no-execute
         * enforcement. These are "const" variables; that qualifier is cast away
        boolean_t ptmp;
        /* Check if the user has requested disabling stack or heap no-execute
         * enforcement. These are "const" variables; that qualifier is cast away


@@ -505,6 +516,7 @@ pmap_bootstrap(
                boolean_t *pdknhp = (boolean_t *) &pmap_disable_kstack_nx;
                *pdknhp = TRUE;
        }
                boolean_t *pdknhp = (boolean_t *) &pmap_disable_kstack_nx;
                *pdknhp = TRUE;
        }

+#endif /* DEVELOPMENT || DEBUG */

 
        boot_args *args = (boot_args *)PE_state.bootArgs;
        if (args->efiMode == kBootArgsEfiMode32) {
 
        boot_args *args = (boot_args *)PE_state.bootArgs;
        if (args->efiMode == kBootArgsEfiMode32) {


@@ -521,17 +533,17 @@ pmap_bootstrap(
         * in the DEBUG kernel) to force the kernel to switch to its own map
         * (and cr3) when control is in kernelspace. The kernel's map does not
         * include (i.e. share) userspace so wild references will cause
         * in the DEBUG kernel) to force the kernel to switch to its own map
         * (and cr3) when control is in kernelspace. The kernel's map does not
         * include (i.e. share) userspace so wild references will cause

-        * a panic. Only copyin and copyout are exempt from this. 
+        * a panic. Only copyin and copyout are exempt from this.

         */
        (void) PE_parse_boot_argn("-no_shared_cr3",
                                  &no_shared_cr3, sizeof (no_shared_cr3));
        if (no_shared_cr3)
                kprintf("Kernel not sharing user map\n");
         */
        (void) PE_parse_boot_argn("-no_shared_cr3",
                                  &no_shared_cr3, sizeof (no_shared_cr3));
        if (no_shared_cr3)
                kprintf("Kernel not sharing user map\n");

-               
+

 #ifdef PMAP_TRACES
        if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
                kprintf("Kernel traces for pmap operations enabled\n");
 #ifdef PMAP_TRACES
        if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
                kprintf("Kernel traces for pmap operations enabled\n");

-       }       
+       }

 #endif /* PMAP_TRACES */
 
 #if MACH_ASSERT
 #endif /* PMAP_TRACES */
 
 #if MACH_ASSERT


@@ -794,11 +806,24 @@ pmap_init(void)
         */
 
        zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);
         */
 
        zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);

+/* TODO: possible general optimisation...pre-allocate via zones commonly created
+ * level3/2 pagetables
+ */
+       pmap_uanchor_zone = zinit(PAGE_SIZE, task_max, PAGE_SIZE, "pagetable user anchors");
+       zone_change(pmap_uanchor_zone, Z_NOENCRYPT, TRUE);
+
+       /* The anchor is required to be page aligned. Zone debugging adds
+        * padding which may violate that requirement. Tell the zone
+        * subsystem that alignment is required.
+        */
+
+       zone_change(pmap_uanchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);

 
        s = (vm_size_t) sizeof(struct pv_hashed_entry);
        pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,
            4096 * 3 /* LCM x86_64*/, "pv_list");
        zone_change(pv_hashed_list_zone, Z_NOENCRYPT, TRUE);
 
        s = (vm_size_t) sizeof(struct pv_hashed_entry);
        pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,
            4096 * 3 /* LCM x86_64*/, "pv_list");
        zone_change(pv_hashed_list_zone, Z_NOENCRYPT, TRUE);

+       zone_change(pv_hashed_list_zone, Z_GZALLOC_EXEMPT, TRUE);

 
        /* create pv entries for kernel pages mapped by low level
           startup code.  these have to exist so we can pmap_remove()
 
        /* create pv entries for kernel pages mapped by low level
           startup code.  these have to exist so we can pmap_remove()


@@ -1273,6 +1298,7 @@ hv_ept_pmap_create(void **ept_pmap, void **eptp)
  *     the map will be used in software only, and
  *     is bounded by that size.
  */
  *     the map will be used in software only, and
  *     is bounded by that size.
  */

+

 pmap_t
 pmap_create_options(
        ledger_t        ledger,
 pmap_t
 pmap_create_options(
        ledger_t        ledger,


@@ -1284,8 +1310,7 @@ pmap_create_options(
        pml4_entry_t    *pml4;
        pml4_entry_t    *kpml4;
 
        pml4_entry_t    *pml4;
        pml4_entry_t    *kpml4;
 

-       PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
-                  (uint32_t) (sz>>32), (uint32_t) sz, flags, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, sz, flags);

 
        size = (vm_size_t) sz;
 
 
        size = (vm_size_t) sz;
 


@@ -1300,24 +1325,20 @@ pmap_create_options(
        /*
         *      Return error when unrecognized flags are passed.
         */
        /*
         *      Return error when unrecognized flags are passed.
         */

-       if ((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0) {
+       if (__improbable((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0)) {

                return(PMAP_NULL);
        }
 
        p = (pmap_t) zalloc(pmap_zone);
        if (PMAP_NULL == p)
                panic("pmap_create zalloc");
                return(PMAP_NULL);
        }
 
        p = (pmap_t) zalloc(pmap_zone);
        if (PMAP_NULL == p)
                panic("pmap_create zalloc");

+

        /* Zero all fields */
        bzero(p, sizeof(*p));
        /* init counts now since we'll be bumping some */
        simple_lock_init(&p->lock, 0);
        /* Zero all fields */
        bzero(p, sizeof(*p));
        /* init counts now since we'll be bumping some */
        simple_lock_init(&p->lock, 0);

-#if 00
-       p->stats.resident_count = 0;
-       p->stats.resident_max = 0;
-       p->stats.wired_count = 0;
-#else

        bzero(&p->stats, sizeof (p->stats));
        bzero(&p->stats, sizeof (p->stats));

-#endif
+

        p->ref_count = 1;
        p->nx_enabled = 1;
        p->pm_shared = FALSE;
        p->ref_count = 1;
        p->nx_enabled = 1;
        p->pm_shared = FALSE;


@@ -1333,10 +1354,13 @@ pmap_create_options(
        }
 
        p->pm_pml4 = zalloc(pmap_anchor_zone);
        }
 
        p->pm_pml4 = zalloc(pmap_anchor_zone);

+       p->pm_upml4 = zalloc(pmap_uanchor_zone); //cleanup for EPT

 
        pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0);
 
        pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0);

+       pmap_assert((((uintptr_t)p->pm_upml4) & PAGE_MASK) == 0);

 
        memset((char *)p->pm_pml4, 0, PAGE_SIZE);
 
        memset((char *)p->pm_pml4, 0, PAGE_SIZE);

+       memset((char *)p->pm_upml4, 0, PAGE_SIZE);

 
        if (flags & PMAP_CREATE_EPT) {
                p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;
 
        if (flags & PMAP_CREATE_EPT) {
                p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;


@@ -1344,6 +1368,7 @@ pmap_create_options(
        } else {
                p->pm_eptp = 0;
                p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
        } else {
                p->pm_eptp = 0;
                p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);

+               p->pm_ucr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_upml4);

        }
 
        /* allocate the vm_objs to hold the pdpt, pde and pte pages */
        }
 
        /* allocate the vm_objs to hold the pdpt, pde and pte pages */


@@ -1367,15 +1392,23 @@ pmap_create_options(
                pml4[KERNEL_PML4_INDEX]    = kpml4[KERNEL_PML4_INDEX];
                pml4[KERNEL_KEXTS_INDEX]   = kpml4[KERNEL_KEXTS_INDEX];
                pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];
                pml4[KERNEL_PML4_INDEX]    = kpml4[KERNEL_PML4_INDEX];
                pml4[KERNEL_KEXTS_INDEX]   = kpml4[KERNEL_KEXTS_INDEX];
                pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];

+               pml4[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];
+#if KASAN
+               pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0];
+               pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1];
+#endif
+               pml4_entry_t    *pml4u = pmap64_user_pml4(p, 0ULL);
+               pml4u[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];

        }
 
 #if MACH_ASSERT
        }
 
 #if MACH_ASSERT

+       p->pmap_stats_assert = TRUE;

        p->pmap_pid = 0;
        strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
 #endif /* MACH_ASSERT */
 
        p->pmap_pid = 0;
        strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
 #endif /* MACH_ASSERT */
 

-       PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
-                  p, flags, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END,
+                  VM_KERNEL_ADDRHIDE(p));

 
        return(p);
 }
 
        return(p);
 }


@@ -1480,6 +1513,34 @@ struct {
        int             purgeable_nonvolatile_compressed_under;
        ledger_amount_t purgeable_nonvolatile_compressed_under_total;
        ledger_amount_t purgeable_nonvolatile_compressed_under_max;
        int             purgeable_nonvolatile_compressed_under;
        ledger_amount_t purgeable_nonvolatile_compressed_under_total;
        ledger_amount_t purgeable_nonvolatile_compressed_under_max;

+
+       int             network_volatile_over;
+       ledger_amount_t network_volatile_over_total;
+       ledger_amount_t network_volatile_over_max;
+       int             network_volatile_under;
+       ledger_amount_t network_volatile_under_total;
+       ledger_amount_t network_volatile_under_max;
+
+       int             network_nonvolatile_over;
+       ledger_amount_t network_nonvolatile_over_total;
+       ledger_amount_t network_nonvolatile_over_max;
+       int             network_nonvolatile_under;
+       ledger_amount_t network_nonvolatile_under_total;
+       ledger_amount_t network_nonvolatile_under_max;
+
+       int             network_volatile_compressed_over;
+       ledger_amount_t network_volatile_compressed_over_total;
+       ledger_amount_t network_volatile_compressed_over_max;
+       int             network_volatile_compressed_under;
+       ledger_amount_t network_volatile_compressed_under_total;
+       ledger_amount_t network_volatile_compressed_under_max;
+
+       int             network_nonvolatile_compressed_over;
+       ledger_amount_t network_nonvolatile_compressed_over_total;
+       ledger_amount_t network_nonvolatile_compressed_over_max;
+       int             network_nonvolatile_compressed_under;
+       ledger_amount_t network_nonvolatile_compressed_under_total;
+       ledger_amount_t network_nonvolatile_compressed_under_max;

 } pmap_ledgers_drift;
 static void pmap_check_ledgers(pmap_t pmap);
 #else /* MACH_ASSERT */
 } pmap_ledgers_drift;
 static void pmap_check_ledgers(pmap_t pmap);
 #else /* MACH_ASSERT */


@@ -1502,7 +1563,7 @@ pmap_destroy(pmap_t       p)
                return;
 
        PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
                return;
 
        PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,

-                  p, 0, 0, 0, 0);
+                  VM_KERNEL_ADDRHIDe(p));

 
        PMAP_LOCK(p);
 
 
        PMAP_LOCK(p);
 


@@ -1525,8 +1586,7 @@ pmap_destroy(pmap_t       p)
        PMAP_UNLOCK(p);
 
        if (c != 0) {
        PMAP_UNLOCK(p);
 
        if (c != 0) {

-               PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
-                          p, 1, 0, 0, 0);
+               PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);

                pmap_assert(p == kernel_pmap);
                return; /* still in use */
        }
                pmap_assert(p == kernel_pmap);
                return; /* still in use */
        }


@@ -1538,6 +1598,7 @@ pmap_destroy(pmap_t       p)
        int inuse_ptepages = 0;
 
        zfree(pmap_anchor_zone, p->pm_pml4);
        int inuse_ptepages = 0;
 
        zfree(pmap_anchor_zone, p->pm_pml4);

+       zfree(pmap_uanchor_zone, p->pm_upml4);

 
        inuse_ptepages += p->pm_obj_pml4->resident_page_count;
        vm_object_deallocate(p->pm_obj_pml4);
 
        inuse_ptepages += p->pm_obj_pml4->resident_page_count;
        vm_object_deallocate(p->pm_obj_pml4);


@@ -1555,8 +1616,7 @@ pmap_destroy(pmap_t       p)
        ledger_dereference(p->ledger);
        zfree(pmap_zone, p);
 
        ledger_dereference(p->ledger);
        zfree(pmap_zone, p);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
-                  0, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);

 }
 
 /*
 }
 
 /*


@@ -1602,7 +1662,15 @@ pmap_protect(
 /*
  *     Set the physical protection on the
  *     specified range of this map as requested.
 /*
  *     Set the physical protection on the
  *     specified range of this map as requested.

- *     Will not increase permissions.
+ *
+ * VERY IMPORTANT: Will *NOT* increase permissions.
+ *     pmap_protect_options() should protect the range against any access types
+ *     that are not in "prot" but it should never grant extra access.
+ *     For example, if "prot" is READ|EXECUTE, that means "remove write
+ *     access" but it does *not* mean "add read and execute" access.
+ *     VM relies on getting soft-faults to enforce extra checks (code
+ *     signing, for example), for example.
+ *     New access permissions are granted via pmap_enter() only.

  */
 void
 pmap_protect_options(
  */
 void
 pmap_protect_options(


@@ -1630,10 +1698,10 @@ pmap_protect_options(
                pmap_remove_options(map, sva, eva, options);
                return;
        }
                pmap_remove_options(map, sva, eva, options);
                return;
        }

+

        PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
        PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,

-                  map,
-                  (uint32_t) (sva >> 32), (uint32_t) sva,
-                  (uint32_t) (eva >> 32), (uint32_t) eva);
+                  VM_KERNEL_ADDRHIDE(map), VM_KERNEL_ADDRHIDE(sva),
+                  VM_KERNEL_ADDRHIDE(eva));

 
        if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled)
                set_NX = FALSE;
 
        if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled)
                set_NX = FALSE;


@@ -1667,26 +1735,26 @@ pmap_protect_options(
                                        continue;
 
                                if (is_ept) {
                                        continue;
 
                                if (is_ept) {

-                                       if (prot & VM_PROT_READ)
-                                               pmap_update_pte(spte, 0, PTE_READ(is_ept));
-                                       else
+                                       if (! (prot & VM_PROT_READ)) {

                                                pmap_update_pte(spte, PTE_READ(is_ept), 0);
                                                pmap_update_pte(spte, PTE_READ(is_ept), 0);

+                                       }

                                }
                                }

-                               if (prot & VM_PROT_WRITE)
-                                       pmap_update_pte(spte, 0, PTE_WRITE(is_ept));
-                               else
+                               if (! (prot & VM_PROT_WRITE)) {

                                        pmap_update_pte(spte, PTE_WRITE(is_ept), 0);
                                        pmap_update_pte(spte, PTE_WRITE(is_ept), 0);

+                               }
+#if DEVELOPMENT || DEBUG
+                               else if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) &&
+                                        map == kernel_pmap) {
+                                       pmap_update_pte(spte, 0, PTE_WRITE(is_ept));
+                               }
+#endif /* DEVELOPMENT || DEBUG */

 
                                if (set_NX) {
 
                                if (set_NX) {

-                                       if (!is_ept)
+                                       if (!is_ept) {

                                                pmap_update_pte(spte, 0, INTEL_PTE_NX);
                                                pmap_update_pte(spte, 0, INTEL_PTE_NX);

-                                       else
+                                       } else {

                                                pmap_update_pte(spte, INTEL_EPT_EX, 0);
                                                pmap_update_pte(spte, INTEL_EPT_EX, 0);

-                               } else {
-                                       if (!is_ept)
-                                               pmap_update_pte(spte, INTEL_PTE_NX, 0);
-                                       else
-                                               pmap_update_pte(spte, 0, INTEL_EPT_EX);
+                                       }

                                }
                                num_found++;
                        }
                                }
                                num_found++;
                        }


@@ -1701,15 +1769,14 @@ pmap_protect_options(
        }
        PMAP_UNLOCK(map);
 
        }
        PMAP_UNLOCK(map);
 

-       PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END,
-                  0, 0, 0, 0, 0);
+       PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);

 
 }
 
 /* Map a (possibly) autogenned block */
 
 }
 
 /* Map a (possibly) autogenned block */

-void
+kern_return_t

 pmap_map_block(
 pmap_map_block(

-       pmap_t          pmap, 
+       pmap_t          pmap,

        addr64_t        va,
        ppnum_t         pa,
        uint32_t        size,
        addr64_t        va,
        ppnum_t         pa,
        uint32_t        size,


@@ -1717,19 +1784,38 @@ pmap_map_block(
        int             attr,
        __unused unsigned int   flags)
 {
        int             attr,
        __unused unsigned int   flags)
 {

+       kern_return_t   kr;
+       addr64_t        original_va = va;

        uint32_t        page;
        int             cur_page_size;
 
        if (attr & VM_MEM_SUPERPAGE)
                cur_page_size =  SUPERPAGE_SIZE;
        uint32_t        page;
        int             cur_page_size;
 
        if (attr & VM_MEM_SUPERPAGE)
                cur_page_size =  SUPERPAGE_SIZE;

-       else 
+       else

                cur_page_size =  PAGE_SIZE;
 
        for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) {
                cur_page_size =  PAGE_SIZE;
 
        for (page = 0; page < size; page+=cur_page_size/PAGE_SIZE) {

-               pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+               kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
+
+               if (kr != KERN_SUCCESS) {
+                       /*
+                        * This will panic for now, as it is unclear that
+                        * removing the mappings is correct.
+                        */
+                       panic("%s: failed pmap_enter, "
+                             "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
+                             __FUNCTION__,
+                             pmap, va, pa, size, prot, flags);
+
+                       pmap_remove(pmap, original_va, va - original_va);
+                       return kr;
+               }
+

                va += cur_page_size;
                pa+=cur_page_size/PAGE_SIZE;
        }
                va += cur_page_size;
                pa+=cur_page_size/PAGE_SIZE;
        }

+
+       return KERN_SUCCESS;

 }
 
 kern_return_t
 }
 
 kern_return_t


@@ -1747,6 +1833,10 @@ pmap_expand_pml4(
 
        DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
 
 
        DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
 

+       /* With the exception of the kext "basement", the kernel's level 4
+        * pagetables must not be dynamically expanded.
+        */
+       assert(map != kernel_pmap || (vaddr == KERNEL_BASEMENT));

        /*
         *      Allocate a VM page for the pml4 page
         */
        /*
         *      Allocate a VM page for the pml4 page
         */


@@ -1812,6 +1902,13 @@ pmap_expand_pml4(
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));

+       pml4_entry_t    *upml4p;
+
+       upml4p = pmap64_user_pml4(map, vaddr);
+       pmap_store_pte(upml4p, pa_to_pte(pa)
+                               | PTE_READ(is_ept)
+                               | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+                               | PTE_WRITE(is_ept));

 
        PMAP_UNLOCK(map);
 
 
        PMAP_UNLOCK(map);
 


@@ -1945,12 +2042,16 @@ pmap_expand(
         * which is for kexts and is in the 512GB immediately below the kernel..
         * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
         */
         * which is for kexts and is in the 512GB immediately below the kernel..
         * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
         */

-       if (map == kernel_pmap && 
-           !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))
-               panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+       if (__improbable(map == kernel_pmap && 
+               !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))) {
+               if ((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0) {
+                       panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+               }
+       }

 
 
        while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
 
 
        while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {

+               assert((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0);

                kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options);
                if (pepkr != KERN_SUCCESS)
                        return pepkr;
                kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options);
                if (pepkr != KERN_SUCCESS)
                        return pepkr;


@@ -2000,7 +2101,7 @@ pmap_expand(
 
                VM_PAGE_FREE(m);
 
 
                VM_PAGE_FREE(m);
 

-               OSAddAtomic(-1,  &inuse_ptepages_count);
+               OSAddAtomic(-1,  &inuse_ptepages_count);//todo replace all with inlines

                PMAP_ZINFO_PFREE(map, PAGE_SIZE);
                return KERN_SUCCESS;
        }
                PMAP_ZINFO_PFREE(map, PAGE_SIZE);
                return KERN_SUCCESS;
        }


@@ -2054,6 +2155,14 @@ pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr)
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));

+
+               pte = pmap64_user_pml4(pmap, vaddr);
+
+               pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
+                               | PTE_READ(is_ept)
+                               | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+                               | PTE_WRITE(is_ept));
+

        }
 
        if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {
        }
 
        if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {


@@ -2362,9 +2471,11 @@ pmap_switch(pmap_t tpmap)
 {
         spl_t  s;
 
 {
         spl_t  s;
 

+       PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(tpmap));

        s = splhigh();          /* Make sure interruptions are disabled */
        set_dirbase(tpmap, current_thread(), cpu_number());
        splx(s);
        s = splhigh();          /* Make sure interruptions are disabled */
        set_dirbase(tpmap, current_thread(), cpu_number());
        splx(s);

+       PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_END);

 }
 
 
 }
 
 


@@ -2426,7 +2537,7 @@ pmap_flush(
 {
        unsigned int    my_cpu;
        unsigned int    cpu;
 {
        unsigned int    my_cpu;
        unsigned int    cpu;

-       unsigned int    cpu_bit;
+       cpumask_t       cpu_bit;

        cpumask_t       cpus_to_respond = 0;
        cpumask_t       cpus_to_signal = 0;
        cpumask_t       cpus_signaled = 0;
        cpumask_t       cpus_to_respond = 0;
        cpumask_t       cpus_to_signal = 0;
        cpumask_t       cpus_signaled = 0;


@@ -2439,7 +2550,7 @@ pmap_flush(
        cpus_to_signal = pfc->pfc_cpus;
 
        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_START,
        cpus_to_signal = pfc->pfc_cpus;
 
        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_START,

-                           NULL, cpus_to_signal, 0, 0, 0);
+                           NULL, cpus_to_signal);

 
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus && cpus_to_signal; cpu++, cpu_bit <<= 1) {
 
 
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus && cpus_to_signal; cpu++, cpu_bit <<= 1) {
 


@@ -2447,7 +2558,7 @@ pmap_flush(
 
                        cpus_to_signal &= ~cpu_bit;
 
 
                        cpus_to_signal &= ~cpu_bit;
 

-                       if (!cpu_datap(cpu)->cpu_running)
+                       if (!cpu_is_running(cpu))

                                continue;
 
                        if (pfc->pfc_invalid_global & cpu_bit)
                                continue;
 
                        if (pfc->pfc_invalid_global & cpu_bit)


@@ -2480,7 +2591,7 @@ pmap_flush(
                deadline = mach_absolute_time() +
                                (TLBTimeOut ? TLBTimeOut : LockTimeOut);
                boolean_t is_timeout_traced = FALSE;
                deadline = mach_absolute_time() +
                                (TLBTimeOut ? TLBTimeOut : LockTimeOut);
                boolean_t is_timeout_traced = FALSE;

-               
+

                /*
                 * Wait for those other cpus to acknowledge
                 */
                /*
                 * Wait for those other cpus to acknowledge
                 */


@@ -2492,7 +2603,7 @@ pmap_flush(
                                 * as appropriate in the PCID case.
                                 */
                                if ((cpus_to_respond & cpu_bit) != 0) {
                                 * as appropriate in the PCID case.
                                 */
                                if ((cpus_to_respond & cpu_bit) != 0) {

-                                       if (!cpu_datap(cpu)->cpu_running ||
+                                       if (!cpu_is_running(cpu) ||

                                            cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
                                            !CPU_CR3_IS_ACTIVE(cpu)) {
                                                cpus_to_respond &= ~cpu_bit;
                                            cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
                                            !CPU_CR3_IS_ACTIVE(cpu)) {
                                                cpus_to_respond &= ~cpu_bit;


@@ -2508,23 +2619,23 @@ pmap_flush(
                                if (TLBTimeOut == 0) {
                                        if (is_timeout_traced)
                                                continue;
                                if (TLBTimeOut == 0) {
                                        if (is_timeout_traced)
                                                continue;

+

                                        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
                                        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),

-                                               NULL, cpus_to_signal, cpus_to_respond, 0, 0);
+                                                           NULL, cpus_to_signal, cpus_to_respond);
+

                                        is_timeout_traced = TRUE;
                                        continue;
                                }
                                        is_timeout_traced = TRUE;
                                        continue;
                                }

-                               pmap_tlb_flush_timeout = TRUE;

                                orig_acks = NMIPI_acks;
                                orig_acks = NMIPI_acks;

-                               mp_cpus_NMIPI(cpus_to_respond);
-
-                               panic("TLB invalidation IPI timeout: "
-                                   "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
-                                   cpus_to_respond, orig_acks, NMIPI_acks);
+                               NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+                               panic("Uninterruptible processor(s): CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+                                     cpus_to_respond, orig_acks, NMIPI_acks, deadline);

                        }
                }
        }
                        }
                }
        }

+

        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_END,
        PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_DELAYED_TLBS) | DBG_FUNC_END,

-                           NULL, cpus_signaled, flush_self, 0, 0);
+                           NULL, cpus_signaled, flush_self);

 
        mp_enable_preemption();
 }
 
        mp_enable_preemption();
 }


@@ -2557,7 +2668,7 @@ void
 pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int options, pmap_flush_context *pfc)
 {
        unsigned int    cpu;
 pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int options, pmap_flush_context *pfc)
 {
        unsigned int    cpu;

-       unsigned int    cpu_bit;
+       cpumask_t       cpu_bit;

        cpumask_t       cpus_to_signal = 0;
        unsigned int    my_cpu = cpu_number();
        pmap_paddr_t    pmap_cr3 = pmap->pm_cr3;
        cpumask_t       cpus_to_signal = 0;
        unsigned int    my_cpu = cpu_number();
        pmap_paddr_t    pmap_cr3 = pmap->pm_cr3;


@@ -2587,7 +2698,8 @@ pmap_flush_tlbs(pmap_t    pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
        }
 
        PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_START,
        }
 
        PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_START,

-                               VM_KERNEL_UNSLIDE_OR_PERM(pmap), options, event_startv, event_endv, 0);
+                           VM_KERNEL_UNSLIDE_OR_PERM(pmap), options,
+                           event_startv, event_endv);

 
        if (is_ept) {
                mp_cpus_call(CPUMASK_ALL, ASYNC, invept, (void*)pmap->pm_eptp);
 
        if (is_ept) {
                mp_cpus_call(CPUMASK_ALL, ASYNC, invept, (void*)pmap->pm_eptp);


@@ -2606,11 +2718,11 @@ pmap_flush_tlbs(pmap_t  pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
                mfence();
        }
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
                mfence();
        }
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {

-               if (!cpu_datap(cpu)->cpu_running)
+               if (!cpu_is_running(cpu))

                        continue;
                uint64_t        cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
                uint64_t        cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);
                        continue;
                uint64_t        cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
                uint64_t        cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);

-
+//recall that the shadowed task cr3 is pre-composed

                if ((pmap_cr3 == cpu_task_cr3) ||
                    (pmap_cr3 == cpu_active_cr3) ||
                    (pmap_is_shared)) {
                if ((pmap_cr3 == cpu_task_cr3) ||
                    (pmap_cr3 == cpu_active_cr3) ||
                    (pmap_is_shared)) {


@@ -2692,7 +2804,7 @@ pmap_flush_tlbs(pmap_t    pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
                                 * as appropriate in the PCID case.
                                 */
                                if ((cpus_to_respond & cpu_bit) != 0) {
                                 * as appropriate in the PCID case.
                                 */
                                if ((cpus_to_respond & cpu_bit) != 0) {

-                                       if (!cpu_datap(cpu)->cpu_running ||
+                                       if (!cpu_is_running(cpu) ||

                                            cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
                                            !CPU_CR3_IS_ACTIVE(cpu)) {
                                                cpus_to_respond &= ~cpu_bit;
                                            cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
                                            !CPU_CR3_IS_ACTIVE(cpu)) {
                                                cpus_to_respond &= ~cpu_bit;


@@ -2709,19 +2821,20 @@ pmap_flush_tlbs(pmap_t  pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
                                        /* cut tracepoint but don't panic */
                                        if (is_timeout_traced)
                                                continue;
                                        /* cut tracepoint but don't panic */
                                        if (is_timeout_traced)
                                                continue;

-                                       PMAP_TRACE_CONSTANT(
-                                               PMAP_CODE(PMAP__FLUSH_TLBS_TO),
-                                               VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, cpus_to_respond, 0, 0);
+
+                                       PMAP_TRACE_CONSTANT(PMAP_CODE(PMAP__FLUSH_TLBS_TO),
+                                                           VM_KERNEL_UNSLIDE_OR_PERM(pmap),
+                                                           cpus_to_signal,
+                                                           cpus_to_respond);
+

                                        is_timeout_traced = TRUE;
                                        continue;
                                }
                                        is_timeout_traced = TRUE;
                                        continue;
                                }

-                               pmap_tlb_flush_timeout = TRUE;

                                orig_acks = NMIPI_acks;
                                orig_acks = NMIPI_acks;

-                               mp_cpus_NMIPI(cpus_to_respond);

 
 

-                               panic("TLB invalidation IPI timeout: "
-                                   "CPU(s) failed to respond to interrupts, unresponsive CPU bitmap: 0x%llx, NMIPI acks: orig: 0x%lx, now: 0x%lx",
-                                   cpus_to_respond, orig_acks, NMIPI_acks);
+                               NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
+                               panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+                                     cpus_to_respond, orig_acks, NMIPI_acks, deadline);

                        }
                }
        }
                        }
                }
        }


@@ -2732,7 +2845,8 @@ pmap_flush_tlbs(pmap_t    pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
 
 out:
        PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_END,
 
 out:
        PMAP_TRACE_CONSTANT(event_code | DBG_FUNC_END,

-                               VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal, event_startv, event_endv, 0);
+                           VM_KERNEL_UNSLIDE_OR_PERM(pmap), cpus_to_signal,
+                           event_startv, event_endv);

 
 }
 
 
 }
 


@@ -2743,16 +2857,9 @@ process_pmap_updates(void)
        pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
        if (pmap_pcid_ncpus) {
                pmap_pcid_validate_current();
        pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
        if (pmap_pcid_ncpus) {
                pmap_pcid_validate_current();

-               if (cpu_datap(ccpu)->cpu_tlb_invalid_global) {
-                       cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
-                       tlb_flush_global();
-               }
-               else {
-                       cpu_datap(ccpu)->cpu_tlb_invalid_local = FALSE;
-                       flush_tlb_raw();
-               }
-       }
-       else {
+               cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
+               tlb_flush_global();
+       } else {

                current_cpu_datap()->cpu_tlb_invalid = FALSE;
                flush_tlb_raw();
        }
                current_cpu_datap()->cpu_tlb_invalid = FALSE;
                flush_tlb_raw();
        }


@@ -2763,14 +2870,12 @@ process_pmap_updates(void)
 void
 pmap_update_interrupt(void)
 {
 void
 pmap_update_interrupt(void)
 {

-        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START,
-                  0, 0, 0, 0, 0);
+        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START);

 
        if (current_cpu_datap()->cpu_tlb_invalid)
                process_pmap_updates();
 
 
        if (current_cpu_datap()->cpu_tlb_invalid)
                process_pmap_updates();
 

-        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END,
-                  0, 0, 0, 0, 0);
+        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END);

 }
 
 #include <mach/mach_vm.h>      /* mach_vm_region_recurse() */
 }
 
 #include <mach/mach_vm.h>      /* mach_vm_region_recurse() */


@@ -2888,6 +2993,8 @@ pmap_permissions_verify(pmap_t ipmap, vm_map_t ivmmap, vm_offset_t sv, vm_offset
 
 #if MACH_ASSERT
 extern int pmap_ledgers_panic;
 
 #if MACH_ASSERT
 extern int pmap_ledgers_panic;

+extern int pmap_ledgers_panic_leeway;
+

 static void
 pmap_check_ledgers(
        pmap_t pmap)
 static void
 pmap_check_ledgers(
        pmap_t pmap)


@@ -2919,248 +3026,57 @@ pmap_check_ledgers(
 
        pmap_ledgers_drift.num_pmaps_checked++;
 
 
        pmap_ledgers_drift.num_pmaps_checked++;
 

-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.phys_footprint,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"phys_footprint\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.phys_footprint_over++;
-                       pmap_ledgers_drift.phys_footprint_over_total += bal;
-                       if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
-                               pmap_ledgers_drift.phys_footprint_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.phys_footprint_under++;
-                       pmap_ledgers_drift.phys_footprint_under_total += bal;
-                       if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
-                               pmap_ledgers_drift.phys_footprint_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.internal,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"internal\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.internal_over++;
-                       pmap_ledgers_drift.internal_over_total += bal;
-                       if (bal > pmap_ledgers_drift.internal_over_max) {
-                               pmap_ledgers_drift.internal_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.internal_under++;
-                       pmap_ledgers_drift.internal_under_total += bal;
-                       if (bal < pmap_ledgers_drift.internal_under_max) {
-                               pmap_ledgers_drift.internal_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.internal_compressed,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"internal_compressed\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.internal_compressed_over++;
-                       pmap_ledgers_drift.internal_compressed_over_total += bal;
-                       if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
-                               pmap_ledgers_drift.internal_compressed_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.internal_compressed_under++;
-                       pmap_ledgers_drift.internal_compressed_under_total += bal;
-                       if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
-                               pmap_ledgers_drift.internal_compressed_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.iokit_mapped,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"iokit_mapped\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.iokit_mapped_over++;
-                       pmap_ledgers_drift.iokit_mapped_over_total += bal;
-                       if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
-                               pmap_ledgers_drift.iokit_mapped_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.iokit_mapped_under++;
-                       pmap_ledgers_drift.iokit_mapped_under_total += bal;
-                       if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
-                               pmap_ledgers_drift.iokit_mapped_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.alternate_accounting,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"alternate_accounting\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.alternate_accounting_over++;
-                       pmap_ledgers_drift.alternate_accounting_over_total += bal;
-                       if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
-                               pmap_ledgers_drift.alternate_accounting_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.alternate_accounting_under++;
-                       pmap_ledgers_drift.alternate_accounting_under_total += bal;
-                       if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
-                               pmap_ledgers_drift.alternate_accounting_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.alternate_accounting_compressed,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"alternate_accounting_compressed\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.alternate_accounting_compressed_over++;
-                       pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
-                       if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
-                               pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.alternate_accounting_compressed_under++;
-                       pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
-                       if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
-                               pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.page_table,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"page_table\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.page_table_over++;
-                       pmap_ledgers_drift.page_table_over_total += bal;
-                       if (bal > pmap_ledgers_drift.page_table_over_max) {
-                               pmap_ledgers_drift.page_table_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.page_table_under++;
-                       pmap_ledgers_drift.page_table_under_total += bal;
-                       if (bal < pmap_ledgers_drift.page_table_under_max) {
-                               pmap_ledgers_drift.page_table_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.purgeable_volatile,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"purgeable_volatile\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.purgeable_volatile_over++;
-                       pmap_ledgers_drift.purgeable_volatile_over_total += bal;
-                       if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
-                               pmap_ledgers_drift.purgeable_volatile_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.purgeable_volatile_under++;
-                       pmap_ledgers_drift.purgeable_volatile_under_total += bal;
-                       if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
-                               pmap_ledgers_drift.purgeable_volatile_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.purgeable_nonvolatile,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"purgeable_nonvolatile\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.purgeable_nonvolatile_over++;
-                       pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
-                       if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
-                               pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.purgeable_nonvolatile_under++;
-                       pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
-                       if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
-                               pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.purgeable_volatile_compressed,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"purgeable_volatile_compressed\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.purgeable_volatile_compressed_over++;
-                       pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
-                       if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
-                               pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.purgeable_volatile_compressed_under++;
-                       pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
-                       if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
-                               pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
-                       }
-               }
-       }
-       ledger_get_balance(pmap->ledger,
-                          task_ledgers.purgeable_nonvolatile_compressed,
-                          &bal);
-       if (bal != 0) {
-               do_panic = TRUE;
-               printf("LEDGER BALANCE proc %d (%s) "
-                      "\"purgeable_nonvolatile_compressed\" = %lld\n",
-                      pid, procname, bal);
-               if (bal > 0) {
-                       pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
-                       pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
-                       if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
-                               pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
-                       }
-               } else {
-                       pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
-                       pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
-                       if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
-                               pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
-                       }
-               }
-       }
+#define LEDGER_CHECK_BALANCE(__LEDGER)                                 \
+MACRO_BEGIN                                                            \
+       int panic_on_negative = TRUE;                                   \
+       ledger_get_balance(pmap->ledger,                                \
+                          task_ledgers.__LEDGER,                       \
+                          &bal);                                       \
+       ledger_get_panic_on_negative(pmap->ledger,                      \
+                                    task_ledgers.__LEDGER,             \
+                                    &panic_on_negative);               \
+       if (bal != 0) {                                                 \
+               if (panic_on_negative ||                                \
+                   (pmap_ledgers_panic &&                              \
+                    pmap_ledgers_panic_leeway > 0 &&                   \
+                    (bal > (pmap_ledgers_panic_leeway * PAGE_SIZE) ||  \
+                     bal < (pmap_ledgers_panic_leeway * PAGE_SIZE)))) { \
+                       do_panic = TRUE;                                \
+               }                                                       \
+               printf("LEDGER BALANCE proc %d (%s) "                   \
+                      "\"%s\" = %lld\n",                               \
+                      pid, procname, #__LEDGER, bal);                  \
+               if (bal > 0) {                                          \
+                       pmap_ledgers_drift.__LEDGER##_over++;           \
+                       pmap_ledgers_drift.__LEDGER##_over_total += bal; \
+                       if (bal > pmap_ledgers_drift.__LEDGER##_over_max) { \
+                               pmap_ledgers_drift.__LEDGER##_over_max = bal; \
+                       }                                               \
+               } else if (bal < 0) {                                   \
+                       pmap_ledgers_drift.__LEDGER##_under++;          \
+                       pmap_ledgers_drift.__LEDGER##_under_total += bal; \
+                       if (bal < pmap_ledgers_drift.__LEDGER##_under_max) { \
+                               pmap_ledgers_drift.__LEDGER##_under_max = bal; \
+                       }                                               \
+               }                                                       \
+       }                                                               \
+MACRO_END
+
+       LEDGER_CHECK_BALANCE(phys_footprint);
+       LEDGER_CHECK_BALANCE(internal);
+       LEDGER_CHECK_BALANCE(internal_compressed);
+       LEDGER_CHECK_BALANCE(iokit_mapped);
+       LEDGER_CHECK_BALANCE(alternate_accounting);
+       LEDGER_CHECK_BALANCE(alternate_accounting_compressed);
+       LEDGER_CHECK_BALANCE(page_table);
+       LEDGER_CHECK_BALANCE(purgeable_volatile);
+       LEDGER_CHECK_BALANCE(purgeable_nonvolatile);
+       LEDGER_CHECK_BALANCE(purgeable_volatile_compressed);
+       LEDGER_CHECK_BALANCE(purgeable_nonvolatile_compressed);
+       LEDGER_CHECK_BALANCE(network_volatile);
+       LEDGER_CHECK_BALANCE(network_nonvolatile);
+       LEDGER_CHECK_BALANCE(network_volatile_compressed);
+       LEDGER_CHECK_BALANCE(network_nonvolatile_compressed);

 
        if (do_panic) {
                if (pmap_ledgers_panic) {
 
        if (do_panic) {
                if (pmap_ledgers_panic) {


@@ -3173,13 +3089,23 @@ pmap_check_ledgers(
        }
 
        if (pmap->stats.resident_count != 0 ||
        }
 
        if (pmap->stats.resident_count != 0 ||

+#if 35156815
+           /*
+            * "wired_count" is unfortunately a bit inaccurate, so let's
+            * tolerate some slight deviation to limit the amount of
+            * somewhat-spurious assertion failures.
+            */
+           pmap->stats.wired_count > 10 ||
+#else /* 35156815 */

            pmap->stats.wired_count != 0 ||
            pmap->stats.wired_count != 0 ||

+#endif /* 35156815 */

            pmap->stats.device != 0 ||
            pmap->stats.internal != 0 ||
            pmap->stats.external != 0 ||
            pmap->stats.reusable != 0 ||
            pmap->stats.compressed != 0) {
            pmap->stats.device != 0 ||
            pmap->stats.internal != 0 ||
            pmap->stats.external != 0 ||
            pmap->stats.reusable != 0 ||
            pmap->stats.compressed != 0) {

-               if (pmap_stats_assert) {
+               if (pmap_stats_assert &&
+                   pmap->pmap_stats_assert) {

                        panic("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
                              pmap, pid, procname,
                              pmap->stats.resident_count,
                        panic("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
                              pmap, pid, procname,
                              pmap->stats.resident_count,


@@ -3214,6 +3140,32 @@ pmap_set_process(
 
        pmap->pmap_pid = pid;
        strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
 
        pmap->pmap_pid = pid;
        strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));

+       if (pmap_ledgers_panic_leeway) {
+               /*
+                * XXX FBDP
+                * Some processes somehow trigger some issues that make
+                * the pmap stats and ledgers go off track, causing
+                * some assertion failures and ledger panics.
+                * Turn off the sanity checks if we allow some ledger leeway
+                * because of that.  We'll still do a final check in
+                * pmap_check_ledgers() for discrepancies larger than the
+                * allowed leeway after the address space has been fully
+                * cleaned up.
+                */
+               pmap->pmap_stats_assert = FALSE;
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.phys_footprint);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.internal);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.internal_compressed);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.iokit_mapped);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.alternate_accounting);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.alternate_accounting_compressed);
+       }

 }
 #endif /* MACH_ASSERT */
 
 }
 #endif /* MACH_ASSERT */
 


@@ -3251,3 +3203,4 @@ void pmap_verify_noncacheable(uintptr_t vaddr) {
                return;
        panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
 }
                return;
        panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
 }

+