#include <vm/vm_kern.h>
#include <vm/vm_map.h>
-#include <i386/lock.h>
+#include <i386/bit_routines.h>
#include <i386/mp_desc.h>
#include <i386/misc_protos.h>
#include <i386/mp.h>
#include <i386/pmap.h>
-#if defined(__i386__)
+#if defined(__i386__) || defined(__x86_64__)
#include <i386/pmap_internal.h>
#endif /* i386 */
#if CONFIG_MCA
}
}
+static void
+cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias)
+{
+ pt_entry_t *pte = NULL;
+
+ /* Require page alignment */
+ assert(page_aligned(gdt));
+ assert(page_aligned(alias));
+
+ pte = pmap_pte(kernel_pmap, alias);
+ pmap_store_pte(pte, kvtophys(gdt) | INTEL_PTE_REF
+ | INTEL_PTE_MOD
+ | INTEL_PTE_WIRED
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE
+ | INTEL_PTE_NX);
+
+ /* TLB flush unneccessry because target processor isn't running yet */
+}
+
void
cpu_desc_init64(cpu_data_t *cdp)
master_ktss64.ist1 = (uintptr_t) low_eintstack
- sizeof(x86_64_intr_stack_frame_t);
- } else {
+ } else if (cdi->cdi_ktss == NULL) { /* Skipping re-init on wake */
cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
+
/*
* Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel
* heap (cpu_desc_table).
* LDT descriptors are mapped into a separate area.
+ * GDT descriptors are addressed by alias to avoid sgdt leaks to user-space.
*/
cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS;
- cdi->cdi_gdt.ptr = (struct fake_descriptor *)cdt->gdt;
+ cdi->cdi_gdt.ptr = (void *)CPU_GDT_ALIAS(cdp->cpu_number);
cdi->cdi_ktss = (void *)&cdt->ktss;
cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top;
cdi->cdi_ldt = cdp->cpu_ldtp;
+ /* Make the virtual alias address for the GDT */
+ cpu_gdt_alias((vm_map_offset_t) &cdt->gdt,
+ (vm_map_offset_t) cdi->cdi_gdt.ptr);
+
/*
* Copy the tables
*/
{
cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
+ /* Stuff the kernel per-cpu data area address into the MSRs */
+ wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
+ wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
+
+ /*
+ * Ensure the TSS segment's busy bit is clear. This is required
+ * for the case of reloading descriptors at wake to avoid
+ * their complete re-initialization.
+ */
+ gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
+
/* Load the GDT, LDT, IDT and TSS */
cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1;
cdi->cdi_idt.size = 0x1000 + cdp->cpu_number;
lldt(KERNEL_LDT);
set_tr(KERNEL_TSS);
- /* Stuff the kernel per-cpu data area address into the MSRs */
- wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
- wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
-
#if GPROF // Hack to enable mcount to work on K64
__asm__ volatile("mov %0, %%gs" : : "rm" ((unsigned short)(KERNEL_DS)));
#endif
/*
* Allocate per-cpu data:
*/
- ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t));
+ ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
printf("cpu_data_alloc() failed, ret=%d\n", ret);
goto abort;
*/
ret = kmem_alloc(kernel_map,
(vm_offset_t *) &cdp->cpu_int_stack_top,
- INTSTACK_SIZE);
+ INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
printf("cpu_data_alloc() int stack failed, ret=%d\n", ret);
goto abort;
*/
ret = kmem_alloc(kernel_map,
(vm_offset_t *) &cdp->cpu_desc_tablep,
- sizeof(cpu_desc_table64_t));
+ sizeof(cpu_desc_table64_t),
+ VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret);
goto abort;
*/
ret = kmem_alloc(kernel_map,
(vm_offset_t *) &cdp->cpu_ldtp,
- sizeof(struct real_descriptor) * LDTSZ);
+ sizeof(struct real_descriptor) * LDTSZ,
+ VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
printf("cpu_data_alloc() ldt failed, ret=%d\n", ret);
goto abort;
real_ncpus++;
simple_unlock(&ncpus_lock);
+ /*
+ * Before this cpu has been assigned a real thread context,
+ * we give it a fake, unique, non-zero thread id which the locking
+ * primitives use as their lock value.
+ * Note that this does not apply to the boot processor, cpu 0, which
+ * transitions to a thread context well before other processors are
+ * started.
+ */
+ cdp->cpu_active_thread = (thread_t) (uintptr_t) cdp->cpu_number;
+
cdp->cpu_nanotime = &pal_rtc_nanotime_info;
kprintf("cpu_data_alloc(%d) %p desc_table: %p "
if (vm_allocate(kernel_map, &vaddr,
(NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE,
- VM_FLAGS_ANYWHERE) != KERN_SUCCESS)
+ VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_CPU)) != KERN_SUCCESS)
panic("cpu_userwindow_init: "
"couldn't allocate user map window");
if (phys_window == 0) {
if (vm_allocate(kernel_map, &phys_window,
- PAGE_SIZE, VM_FLAGS_ANYWHERE)
+ PAGE_SIZE, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_CPU))
!= KERN_SUCCESS)
panic("cpu_physwindow_init: "
"couldn't allocate phys map window");
cpu_data_t *cdp;
boolean_t istate;
- ret = kmem_alloc(kernel_map, &istk, INTSTACK_SIZE);
+ ret = kmem_alloc(kernel_map, &istk, INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
panic("cpu_data_realloc() stack alloc, ret=%d\n", ret);
}
bzero((void*) istk, INTSTACK_SIZE);
istk += INTSTACK_SIZE;
- ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t));
+ ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
panic("cpu_data_realloc() cpu data alloc, ret=%d\n", ret);
}
timer_call_queue_init(&cdp->rtclock_timer.queue);
/* Allocate the separate fault stack */
- ret = kmem_alloc(kernel_map, &fstk, PAGE_SIZE);
+ ret = kmem_alloc(kernel_map, &fstk, PAGE_SIZE, VM_KERN_MEMORY_CPU);
if (ret != KERN_SUCCESS) {
panic("cpu_data_realloc() fault stack alloc, ret=%d\n", ret);
}
projects / apple / xnu.git / blobdiff