+/*
+ * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()
+ * due to the mutation of physfree.
+ */
+static void *
+ALLOCPAGES(int npages)
+{
+ uintptr_t tmp = (uintptr_t)physfree;
+ bzero(physfree, npages * PAGE_SIZE);
+ physfree += npages * PAGE_SIZE;
+#ifdef __x86_64__
+ tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
+#endif
+ return (void *)tmp;
+}
+
+static void
+fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
+{
+ int i;
+ for (i=0; i<count; i++) {
+ base[index] = src | prot | INTEL_PTE_VALID;
+ src += PAGE_SIZE;
+ index++;
+ }
+}
+
+extern pmap_paddr_t first_avail;
+
+#ifdef __x86_64__
+int break_kprintf = 0;
+
+uint64_t
+x86_64_pre_sleep(void)
+{
+ IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
+ uint64_t oldcr3 = get_cr3_raw();
+ set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
+ return oldcr3;
+}
+
+void
+x86_64_post_sleep(uint64_t new_cr3)
+{
+ IdlePML4[0] = 0;
+ set_cr3_raw((uint32_t) new_cr3);
+}
+
+#endif
+
+#ifdef __i386__
+#define ID_MAP_VTOP(x) x
+#endif
+
+
+#ifdef __x86_64__
+// Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address
+// NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
+// the PCI hole (which is less 4GB but not more).
+
+// Compile-time guard:
+extern int maxphymapsupported[NPHYSMAP <= PTE_PER_PAGE ? 1 : -1];
+static void
+physmap_init(void)
+{
+ pt_entry_t *physmapL3 = ALLOCPAGES(1);
+ struct {
+ pt_entry_t entries[PTE_PER_PAGE];
+ } * physmapL2 = ALLOCPAGES(NPHYSMAP);
+
+ uintptr_t i;
+ for(i=0;i<NPHYSMAP;i++) {
+ physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(&physmapL2[i]))
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE;
+ uintptr_t j;
+ for(j=0;j<PTE_PER_PAGE;j++) {
+ physmapL2[i].entries[j] = (((i*PTE_PER_PAGE+j)<<PDSHIFT)
+ | INTEL_PTE_PS
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE);
+ }
+ }
+
+ IdlePML4[KERNEL_PHYSMAP_INDEX] = ((uintptr_t)ID_MAP_VTOP(physmapL3))
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE;
+ if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) {
+ IdlePML4[KERNEL_PHYSMAP_INDEX] |= INTEL_PTE_NX;
+ }
+
+ DBG("physical map idlepml4[%d]: 0x%llx\n",
+ KERNEL_PHYSMAP_INDEX, IdlePML4[KERNEL_PHYSMAP_INDEX]);
+}
+#endif
+
+static void
+Idle_PTs_init(void)
+{
+ /* Allocate the "idle" kernel page tables: */
+ KPTphys = ALLOCPAGES(NKPT); /* level 1 */
+ IdlePTD = ALLOCPAGES(NPGPTD); /* level 2 */
+
+#ifdef __x86_64__
+ physmap_init();
+#else
+ IdlePDPT64 = ALLOCPAGES(1);
+
+ // Recursive mapping of PTEs
+ fillkpt(IdlePTD, INTEL_PTE_WRITE, (uintptr_t)IdlePTD, PTDPTDI, NPGPTD);
+ // commpage
+ fillkpt(IdlePTD, INTEL_PTE_WRITE|INTEL_PTE_USER, (uintptr_t)ALLOCPAGES(1), _COMM_PAGE32_BASE_ADDRESS >> PDESHIFT,1);
+#endif
+ // Fill the lowest level with everything up to physfree
+ fillkpt(KPTphys,
+ INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
+
+ // Rewrite the 2nd-lowest level to point to pages of KPTphys.
+ // This was previously filled statically by idle_pt.c, and thus
+ // must be done after the KPTphys fill since IdlePTD is in use
+ fillkpt(IdlePTD,
+ INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
+
+ // IdlePDPT entries
+#ifdef __i386__
+ fillkpt(IdlePDPT, 0, (uintptr_t)IdlePTD, 0, NPGPTD);
+#else
+ fillkpt(IdlePDPT, INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
+#endif
+
+ // Flush the TLB now we're done rewriting the page tables..
+ set_cr3_raw(get_cr3_raw());
+}
+
+/*
+ * vstart() is called in the natural mode (64bit for K64, 32 for K32)
+ * on a set of bootstrap pagetables which use large, 2MB pages to map
+ * all of physical memory in both. See idle_pt.c for details.
+ *
+ * In K64 this identity mapping is mirrored the top and bottom 512GB
+ * slots of PML4.
+ *
+ * The bootstrap processor called with argument boot_args_start pointing to
+ * the boot-args block. The kernel's (4K page) page tables are allocated and
+ * initialized before switching to these.
+ *
+ * Non-bootstrap processors are called with argument boot_args_start NULL.
+ * These processors switch immediately to the existing kernel page tables.
+ */
+void
+vstart(vm_offset_t boot_args_start)
+{
+ boolean_t is_boot_cpu = !(boot_args_start == 0);
+ int cpu;
+ uint32_t lphysfree;
+
+ postcode(VSTART_ENTRY);
+
+ if (is_boot_cpu) {
+ /*
+ * Get startup parameters.
+ */
+ kernelBootArgs = (boot_args *)boot_args_start;
+ lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
+ physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1));
+#if DEBUG
+ pal_serial_init();
+#endif
+ DBG("revision 0x%x\n", kernelBootArgs->Revision);
+ DBG("version 0x%x\n", kernelBootArgs->Version);
+ DBG("command line %s\n", kernelBootArgs->CommandLine);
+ DBG("memory map 0x%x\n", kernelBootArgs->MemoryMap);
+ DBG("memory map sz 0x%x\n", kernelBootArgs->MemoryMapSize);
+ DBG("kaddr 0x%x\n", kernelBootArgs->kaddr);
+ DBG("ksize 0x%x\n", kernelBootArgs->ksize);
+ DBG("physfree %p\n", physfree);
+ DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
+ kernelBootArgs,
+ &kernelBootArgs->ksize,
+ &kernelBootArgs->kaddr);
+#ifdef __x86_64__
+ /* enable NX/XD, boot processor */
+ if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) {
+ wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_NXE);
+ DBG("vstart() NX/XD enabled\n");
+ }
+#endif
+ postcode(PSTART_PAGE_TABLES);
+
+ Idle_PTs_init();
+
+ first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
+
+ cpu = 0;
+ cpu_data_alloc(TRUE);
+
+
+ /*
+ * Setup boot args given the physical start address.
+ */
+ kernelBootArgs = (boot_args *)
+ ml_static_ptovirt(boot_args_start);
+ DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
+ (unsigned long)boot_args_start, kernelBootArgs);
+
+ PE_init_platform(FALSE, kernelBootArgs);
+ postcode(PE_INIT_PLATFORM_D);
+ } else {
+ /* Find our logical cpu number */
+ cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
+ DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
+#ifdef __x86_64__
+ if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) {
+ wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_NXE);
+ DBG("vstart() NX/XD enabled, non-boot\n");
+ }
+#endif
+ }
projects / apple / xnu.git / blobdiff