+#define PTE_PER_PAGE 512 /* number of PTE's per page on any level */
+
+ /* cleanly define parameters for all the page table levels */
+typedef uint64_t pml4_entry_t;
+#define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t)))
+#define PML4SHIFT 39
+#define PML4PGSHIFT 9
+#define NBPML4 (1ULL << PML4SHIFT)
+#define PML4MASK (NBPML4-1)
+#define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
+
+typedef uint64_t pdpt_entry_t;
+#define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t)))
+#define PDPTSHIFT 30
+#define PDPTPGSHIFT 9
+#define NBPDPT (1 << PDPTSHIFT)
+#define PDPTMASK (NBPDPT-1)
+#define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
+
+typedef uint64_t pd_entry_t;
+#define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t)))
+#define PDSHIFT 21
+#define PDPGSHIFT 9
+#define NBPD (1 << PDSHIFT)
+#define PDMASK (NBPD-1)
+#define PD_ENTRY_NULL ((pd_entry_t *) 0)
+
+typedef uint64_t pt_entry_t;
+#define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t)))
+#define PTSHIFT 12
+#define PTPGSHIFT 9
+#define NBPT (1 << PTSHIFT)
+#define PTMASK (NBPT-1)
+#define PT_ENTRY_NULL ((pt_entry_t *) 0)
+
+typedef uint64_t pmap_paddr_t;
+
+/* superpages */
+#ifdef __x86_64__
+#define SUPERPAGE_NBASEPAGES 512
+#else
+#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
+#endif
+
+/*
+ * Atomic 64-bit store of a page table entry.
+ */
+static inline void
+pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
+{
+#ifdef __i386__
+ /*
+ * Load the new value into %ecx:%ebx
+ * Load the old value into %edx:%eax
+ * Compare-exchange-8bytes at address entryp (loaded in %edi)
+ * If the compare succeeds, the new value will have been stored.
+ * Otherwise, the old value changed and reloaded, so try again.
+ */
+ __asm__ volatile(
+ " movl (%0), %%eax \n\t"
+ " movl 4(%0), %%edx \n\t"
+ "1: \n\t"
+ " cmpxchg8b (%0) \n\t"
+ " jnz 1b"
+ :
+ : "D" (entryp),
+ "b" ((uint32_t)value),
+ "c" ((uint32_t)(value >> 32))
+ : "eax", "edx", "memory");
+#else
+ /*
+ * In the 32-bit kernel a compare-and-exchange loop was
+ * required to provide atomicity. For K64, life is easier:
+ */
+ *entryp = value;
+#endif
+}
+
+/*
+ * Atomic 64-bit compare and exchange of a page table entry.
+ */
+static inline boolean_t
+pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
+{
+ boolean_t ret;
+
+#ifdef __i386__
+ /*
+ * Load the old value into %edx:%eax
+ * Load the new value into %ecx:%ebx
+ * Compare-exchange-8bytes at address entryp (loaded in %edi)
+ * If the compare succeeds, the new value is stored, return TRUE.
+ * Otherwise, no swap is made, return FALSE.
+ */
+ asm volatile(
+ " lock; cmpxchg8b (%1) \n\t"
+ " setz %%al \n\t"
+ " movzbl %%al,%0"
+ : "=a" (ret)
+ : "D" (entryp),
+ "a" ((uint32_t)old),
+ "d" ((uint32_t)(old >> 32)),
+ "b" ((uint32_t)new),
+ "c" ((uint32_t)(new >> 32))
+ : "memory");
+#else
+ /*
+ * Load the old value into %rax
+ * Load the new value into another register
+ * Compare-exchange-quad at address entryp
+ * If the compare succeeds, the new value is stored, return TRUE.
+ * Otherwise, no swap is made, return FALSE.
+ */
+ asm volatile(
+ " lock; cmpxchgq %2,(%3) \n\t"
+ " setz %%al \n\t"
+ " movzbl %%al,%0"
+ : "=a" (ret)
+ : "a" (old),
+ "r" (new),
+ "r" (entryp)
+ : "memory");
+#endif
+ return ret;
+}
+
+#define pmap_update_pte(entryp, old, new) \
+ while (!pmap_cmpx_pte((entryp), (old), (new)))
+
+
+/* in 64 bit spaces, the number of each type of page in the page tables */
+#define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
+#define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
+#define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
+#define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
+
+#ifdef __i386__
+/*
+ * The 64-bit kernel is remapped in uber-space which is at the base
+ * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
+ * 512GB from the top of virtual space (or zero).
+ */
+#define KERNEL_UBER_PML4_INDEX 511
+#define KERNEL_UBER_BASE (0ULL - NBPML4)
+#define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32))
+#else
+#define KERNEL_PML4_INDEX 511
+#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */
+#define KERNEL_PHYSMAP_INDEX 509 /* virtual to physical map */
+#define KERNEL_BASE (0ULL - NBPML4)
+#define KERNEL_BASEMENT (KERNEL_BASE - NBPML4)
+#endif
+
projects / apple / xnu.git / blobdiff