+
+#if MACH_ASSERT
+extern int pmap_ledgers_panic;
+static void
+pmap_check_ledgers(
+ pmap_t pmap)
+{
+ ledger_amount_t bal;
+ int pid;
+ char *procname;
+ boolean_t do_panic;
+
+ if (pmap->pmap_pid == 0) {
+ /*
+ * This pmap was not or is no longer fully associated
+ * with a task (e.g. the old pmap after a fork()/exec() or
+ * spawn()). Its "ledger" still points at a task that is
+ * now using a different (and active) address space, so
+ * we can't check that all the pmap ledgers are balanced here.
+ *
+ * If the "pid" is set, that means that we went through
+ * pmap_set_process() in task_terminate_internal(), so
+ * this task's ledger should not have been re-used and
+ * all the pmap ledgers should be back to 0.
+ */
+ return;
+ }
+
+ do_panic = FALSE;
+ pid = pmap->pmap_pid;
+ procname = pmap->pmap_procname;
+
+ pmap_ledgers_drift.num_pmaps_checked++;
+
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.phys_footprint,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"phys_footprint\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.phys_footprint_over++;
+ pmap_ledgers_drift.phys_footprint_over_total += bal;
+ if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
+ pmap_ledgers_drift.phys_footprint_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.phys_footprint_under++;
+ pmap_ledgers_drift.phys_footprint_under_total += bal;
+ if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
+ pmap_ledgers_drift.phys_footprint_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.internal,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"internal\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.internal_over++;
+ pmap_ledgers_drift.internal_over_total += bal;
+ if (bal > pmap_ledgers_drift.internal_over_max) {
+ pmap_ledgers_drift.internal_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.internal_under++;
+ pmap_ledgers_drift.internal_under_total += bal;
+ if (bal < pmap_ledgers_drift.internal_under_max) {
+ pmap_ledgers_drift.internal_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.internal_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"internal_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.internal_compressed_over++;
+ pmap_ledgers_drift.internal_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
+ pmap_ledgers_drift.internal_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.internal_compressed_under++;
+ pmap_ledgers_drift.internal_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
+ pmap_ledgers_drift.internal_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.iokit_mapped,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"iokit_mapped\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.iokit_mapped_over++;
+ pmap_ledgers_drift.iokit_mapped_over_total += bal;
+ if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
+ pmap_ledgers_drift.iokit_mapped_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.iokit_mapped_under++;
+ pmap_ledgers_drift.iokit_mapped_under_total += bal;
+ if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
+ pmap_ledgers_drift.iokit_mapped_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.alternate_accounting,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"alternate_accounting\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.alternate_accounting_over++;
+ pmap_ledgers_drift.alternate_accounting_over_total += bal;
+ if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
+ pmap_ledgers_drift.alternate_accounting_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.alternate_accounting_under++;
+ pmap_ledgers_drift.alternate_accounting_under_total += bal;
+ if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
+ pmap_ledgers_drift.alternate_accounting_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.alternate_accounting_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"alternate_accounting_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.alternate_accounting_compressed_over++;
+ pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
+ pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.alternate_accounting_compressed_under++;
+ pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
+ pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.page_table,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"page_table\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.page_table_over++;
+ pmap_ledgers_drift.page_table_over_total += bal;
+ if (bal > pmap_ledgers_drift.page_table_over_max) {
+ pmap_ledgers_drift.page_table_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.page_table_under++;
+ pmap_ledgers_drift.page_table_under_total += bal;
+ if (bal < pmap_ledgers_drift.page_table_under_max) {
+ pmap_ledgers_drift.page_table_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_volatile,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_volatile\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_volatile_over++;
+ pmap_ledgers_drift.purgeable_volatile_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
+ pmap_ledgers_drift.purgeable_volatile_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_volatile_under++;
+ pmap_ledgers_drift.purgeable_volatile_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
+ pmap_ledgers_drift.purgeable_volatile_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_nonvolatile,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_nonvolatile\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_nonvolatile_over++;
+ pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_nonvolatile_under++;
+ pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_volatile_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_volatile_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_over++;
+ pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_volatile_compressed_under++;
+ pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
+ pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
+ }
+ }
+ }
+ ledger_get_balance(pmap->ledger,
+ task_ledgers.purgeable_nonvolatile_compressed,
+ &bal);
+ if (bal != 0) {
+ do_panic = TRUE;
+ printf("LEDGER BALANCE proc %d (%s) "
+ "\"purgeable_nonvolatile_compressed\" = %lld\n",
+ pid, procname, bal);
+ if (bal > 0) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
+ if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
+ }
+ } else {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
+ if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
+ pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
+ }
+ }
+ }
+
+ if (do_panic) {
+ if (pmap_ledgers_panic) {
+ panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
+ pmap, pid, procname);
+ } else {
+ printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
+ pmap, pid, procname);
+ }
+ }
+
+ if (pmap->stats.resident_count != 0 ||
+ pmap->stats.wired_count != 0 ||
+ pmap->stats.device != 0 ||
+ pmap->stats.internal != 0 ||
+ pmap->stats.external != 0 ||
+ pmap->stats.reusable != 0 ||
+ pmap->stats.compressed != 0) {
+ if (pmap_stats_assert) {
+ panic("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
+ pmap, pid, procname,
+ pmap->stats.resident_count,
+ pmap->stats.wired_count,
+ pmap->stats.device,
+ pmap->stats.internal,
+ pmap->stats.external,
+ pmap->stats.reusable,
+ pmap->stats.compressed);
+ } else {
+ printf("pmap_destroy(%p) %d[%s] imbalanced stats: resident=%d wired=%d device=%d internal=%d external=%d reusable=%d compressed=%lld",
+ pmap, pid, procname,
+ pmap->stats.resident_count,
+ pmap->stats.wired_count,
+ pmap->stats.device,
+ pmap->stats.internal,
+ pmap->stats.external,
+ pmap->stats.reusable,
+ pmap->stats.compressed);
+ }
+ }
+}
+
+void
+pmap_set_process(
+ pmap_t pmap,
+ int pid,
+ char *procname)
+{
+ if (pmap == NULL)
+ return;
+
+ pmap->pmap_pid = pid;
+ strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
+}
+#endif /* MACH_ASSERT */
+
+
+#if DEVELOPMENT || DEBUG
+int pmap_pagezero_mitigation = 1;
+#endif
+
+void pmap_advise_pagezero_range(pmap_t lpmap, uint64_t low_bound) {
+#if DEVELOPMENT || DEBUG
+ if (pmap_pagezero_mitigation == 0) {
+ lpmap->pagezero_accessible = FALSE;
+ return;
+ }
+#endif
+ lpmap->pagezero_accessible = ((pmap_smap_enabled == FALSE) && (low_bound < 0x1000));
+ if (lpmap == current_pmap()) {
+ mp_disable_preemption();
+ current_cpu_datap()->cpu_pagezero_mapped = lpmap->pagezero_accessible;
+ mp_enable_preemption();
+ }
+}
+
+void pmap_verify_noncacheable(uintptr_t vaddr) {
+ pt_entry_t *ptep = NULL;
+ ptep = pmap_pte(kernel_pmap, vaddr);
+ if (ptep == NULL) {
+ panic("pmap_verify_noncacheable: no translation for 0x%lx", vaddr);
+ }
+ /* Non-cacheable OK */
+ if (*ptep & (INTEL_PTE_NCACHE))
+ return;
+ /* Write-combined OK */
+ if (*ptep & (INTEL_PTE_PTA))
+ return;
+ panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
+}
projects / apple / xnu.git / blobdiff