+static void
+cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias)
+{
+ pt_entry_t *pte = NULL;
+
+ /* Require page alignment */
+ assert(page_aligned(gdt));
+ assert(page_aligned(alias));
+
+ pte = pmap_pte(kernel_pmap, alias);
+ pmap_store_pte(pte, kvtophys(gdt) | INTEL_PTE_REF
+ | INTEL_PTE_MOD
+ | INTEL_PTE_WIRED
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE
+ | INTEL_PTE_NX);
+
+ /* TLB flush unneccessry because target processor isn't running yet */
+}
+
+
+void
+cpu_desc_init64(cpu_data_t *cdp)
+{
+ cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
+
+ if (cdp == &cpu_data_master) {
+ /*
+ * Master CPU uses the tables built at boot time.
+ * Just set the index pointers to the low memory space.
+ */
+ cdi->cdi_ktss = (void *)&master_ktss64;
+ cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
+ cdi->cdi_gdt.ptr = (void *)MASTER_GDT_ALIAS;
+ cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS;
+ cdi->cdi_ldt = (struct fake_descriptor *) master_ldt;
+
+ /* Replace the expanded LDTs and TSS slots in the GDT */
+ kernel_ldt_desc64.offset64 = (uintptr_t) &master_ldt;
+ *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
+ kernel_ldt_desc64;
+ *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] =
+ kernel_ldt_desc64;
+ kernel_tss_desc64.offset64 = (uintptr_t) &master_ktss64;
+ *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
+ kernel_tss_desc64;
+
+ /* Fix up the expanded descriptors for 64-bit. */
+ fix_desc64((void *) &master_idt64, IDTSZ);
+ fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
+ fix_desc64((void *) &master_gdt[sel_idx(USER_LDT)], 1);
+ fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
+
+ /*
+ * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS
+ * Note: this will be dynamically re-allocated in VM later.
+ */
+ master_ktss64.ist2 = (uintptr_t) low_eintstack;
+ master_ktss64.ist1 = (uintptr_t) low_eintstack
+ - sizeof(x86_64_intr_stack_frame_t);
+
+ } else if (cdi->cdi_ktss == NULL) { /* Skipping re-init on wake */
+ cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
+
+ /*
+ * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel
+ * heap (cpu_desc_table).
+ * LDT descriptors are mapped into a separate area.
+ * GDT descriptors are addressed by alias to avoid sgdt leaks to user-space.
+ */
+ cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS;
+ cdi->cdi_gdt.ptr = (void *)CPU_GDT_ALIAS(cdp->cpu_number);
+ cdi->cdi_ktss = (void *)&cdt->ktss;
+ cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top;
+ cdi->cdi_ldt = cdp->cpu_ldtp;
+
+ /* Make the virtual alias address for the GDT */
+ cpu_gdt_alias((vm_map_offset_t) &cdt->gdt,
+ (vm_map_offset_t) cdi->cdi_gdt.ptr);
+
+ /*
+ * Copy the tables
+ */
+ bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt));
+ bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, sizeof(master_ldt));
+ bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss));
+
+ /*
+ * Fix up the entries in the GDT to point to
+ * this LDT and this TSS.
+ */
+ kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
+ *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
+ kernel_ldt_desc64;
+ fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
+
+ kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
+ *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
+ kernel_ldt_desc64;
+ fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
+
+ kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktss;
+ *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
+ kernel_tss_desc64;
+ fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
+
+ /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */
+ bzero((void *) cdt->fstk, sizeof(cdt->fstk));
+ cdt->ktss.ist2 = (unsigned long)cdt->fstk + sizeof(cdt->fstk);
+ cdt->ktss.ist1 = cdt->ktss.ist2
+ - sizeof(x86_64_intr_stack_frame_t);
+ }
+
+ /* Require that the top of the sysenter stack is 16-byte aligned */
+ if ((cdi->cdi_sstk % 16) != 0)
+ panic("cpu_desc_init64() sysenter stack not 16-byte aligned");
+}
+
+
+void
+cpu_desc_load64(cpu_data_t *cdp)
+{
+ cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
+
+ /* Stuff the kernel per-cpu data area address into the MSRs */
+ wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
+ wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
+
+ /*
+ * Ensure the TSS segment's busy bit is clear. This is required
+ * for the case of reloading descriptors at wake to avoid
+ * their complete re-initialization.
+ */
+ gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
+
+ /* Load the GDT, LDT, IDT and TSS */
+ cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1;
+ cdi->cdi_idt.size = 0x1000 + cdp->cpu_number;
+ lgdt((uintptr_t *) &cdi->cdi_gdt);
+ lidt((uintptr_t *) &cdi->cdi_idt);
+ lldt(KERNEL_LDT);
+ set_tr(KERNEL_TSS);
+
+#if GPROF // Hack to enable mcount to work on K64
+ __asm__ volatile("mov %0, %%gs" : : "rm" ((unsigned short)(KERNEL_DS)));
+#endif
+}
+
+
+/*
+ * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
+ */
+static void
+fast_syscall_init64(__unused cpu_data_t *cdp)
+{
+ wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS);
+ wrmsr64(MSR_IA32_SYSENTER_EIP, (uintptr_t) hi64_sysenter);
+ wrmsr64(MSR_IA32_SYSENTER_ESP, current_sstk());
+ /* Enable syscall/sysret */
+ wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
+
+ /*
+ * MSRs for 64-bit syscall/sysret
+ * Note USER_CS because sysret uses this + 16 when returning to
+ * 64-bit code.
+ */
+ wrmsr64(MSR_IA32_LSTAR, (uintptr_t) hi64_syscall);
+ wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) |
+ (((uint64_t)KERNEL64_CS) << 32));
+ /*
+ * Emulate eflags cleared by sysenter but note that
+ * we also clear the trace trap to avoid the complications
+ * of single-stepping into a syscall. The nested task bit
+ * is also cleared to avoid a spurious "task switch"
+ * should we choose to return via an IRET.
+ */
+ wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
+
+}
+
+
projects / apple / xnu.git / blobdiff