/*
- * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#if DEBUG
#include <machine/pal_routines.h>
#endif
+
+#if MONOTONIC
+#include <kern/monotonic.h>
+#endif /* MONOTONIC */
+
+#include <san/kasan.h>
+
#if DEBUG
#define DBG(x...) kprintf(x)
#else
extern const char version_variant[];
extern int nx_enabled;
-uint64_t physmap_base, physmap_max;
+/*
+ * Set initial values so that ml_phys_* routines can use the booter's ID mapping
+ * to touch physical space before the kernel's physical aperture exists.
+ */
+uint64_t physmap_base = 0;
+uint64_t physmap_max = 4*GB;
pd_entry_t *KPTphys;
pd_entry_t *IdlePTD;
pml4_entry_t *IdlePML4;
char *physfree;
+void idt64_remap(void);
/*
* Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()
KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
}
-static void
-descriptor_alias_init()
-{
- vm_offset_t master_gdt_phys;
- vm_offset_t master_gdt_alias_phys;
- vm_offset_t master_idt_phys;
- vm_offset_t master_idt_alias_phys;
-
- assert(((vm_offset_t)master_gdt & PAGE_MASK) == 0);
- assert(((vm_offset_t)master_idt64 & PAGE_MASK) == 0);
-
- master_gdt_phys = (vm_offset_t) ID_MAP_VTOP(master_gdt);
- master_idt_phys = (vm_offset_t) ID_MAP_VTOP(master_idt64);
- master_gdt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_GDT_ALIAS);
- master_idt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_IDT_ALIAS);
-
- DBG("master_gdt_phys: %p\n", (void *) master_gdt_phys);
- DBG("master_idt_phys: %p\n", (void *) master_idt_phys);
- DBG("master_gdt_alias_phys: %p\n", (void *) master_gdt_alias_phys);
- DBG("master_idt_alias_phys: %p\n", (void *) master_idt_alias_phys);
-
- KPTphys[atop_kernel(master_gdt_alias_phys)] = master_gdt_phys |
- INTEL_PTE_VALID | INTEL_PTE_NX | INTEL_PTE_WRITE;
- KPTphys[atop_kernel(master_idt_alias_phys)] = master_idt_phys |
- INTEL_PTE_VALID | INTEL_PTE_NX; /* read-only */
-}
+void doublemap_init(void);
static void
Idle_PTs_init(void)
postcode(VSTART_PHYSMAP_INIT);
physmap_init();
-
- postcode(VSTART_DESC_ALIAS_INIT);
-
- descriptor_alias_init();
+ doublemap_init();
+ idt64_remap();
postcode(VSTART_SET_CR3);
}
+extern void vstart_trap_handler;
+
+#define BOOT_TRAP_VECTOR(t) \
+ [t] = { \
+ (uintptr_t) &vstart_trap_handler, \
+ KERNEL64_CS, \
+ 0, \
+ ACC_P|ACC_PL_K|ACC_INTR_GATE, \
+ 0 \
+ },
+
+/* Recursive macro to iterate 0..31 */
+#define L0(x,n) x(n)
+#define L1(x,n) L0(x,n-1) L0(x,n)
+#define L2(x,n) L1(x,n-2) L1(x,n)
+#define L3(x,n) L2(x,n-4) L2(x,n)
+#define L4(x,n) L3(x,n-8) L3(x,n)
+#define L5(x,n) L4(x,n-16) L4(x,n)
+#define FOR_0_TO_31(x) L5(x,31)
+
+/*
+ * Bootstrap IDT. Active only during early startup.
+ * Only the trap vectors are defined since interrupts are masked.
+ * All traps point to a common handler.
+ */
+struct fake_descriptor64 master_boot_idt64[IDTSZ]
+ __attribute__((section("__HIB,__desc")))
+ __attribute__((aligned(PAGE_SIZE))) = {
+ FOR_0_TO_31(BOOT_TRAP_VECTOR)
+};
+
+static void
+vstart_idt_init(void)
+{
+ x86_64_desc_register_t vstart_idt = {
+ sizeof(master_boot_idt64),
+ master_boot_idt64 };
+
+ fix_desc64(master_boot_idt64, 32);
+ lidt((void *)&vstart_idt);
+}
/*
* vstart() is called in the natural mode (64bit for K64, 32 for K32)
* Non-bootstrap processors are called with argument boot_args_start NULL.
* These processors switch immediately to the existing kernel page tables.
*/
+__attribute__((noreturn))
void
vstart(vm_offset_t boot_args_start)
{
boolean_t is_boot_cpu = !(boot_args_start == 0);
- int cpu;
+ int cpu = 0;
uint32_t lphysfree;
postcode(VSTART_ENTRY);
if (is_boot_cpu) {
+ /*
+ * Set-up temporary trap handlers during page-table set-up.
+ */
+ vstart_idt_init();
+ postcode(VSTART_IDT_INIT);
+
/*
* Get startup parameters.
*/
ml_static_ptovirt(boot_args_start);
DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
(unsigned long)boot_args_start, kernelBootArgs);
+
+#if KASAN
+ kasan_reserve_memory(kernelBootArgs);
+#endif
+
PE_init_platform(FALSE, kernelBootArgs);
postcode(PE_INIT_PLATFORM_D);
Idle_PTs_init();
postcode(VSTART_IDLE_PTS_INIT);
+#if KASAN
+ /* Init kasan and map whatever was stolen from physfree */
+ kasan_init();
+ kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
+#endif
+
+#if MONOTONIC
+ mt_early_init();
+#endif /* MONOTONIC */
+
first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
- cpu = 0;
cpu_data_alloc(TRUE);
+
+ cpu_desc_init(cpu_datap(0));
+ postcode(VSTART_CPU_DESC_INIT);
+ cpu_desc_load(cpu_datap(0));
+
+ postcode(VSTART_CPU_MODE_INIT);
+ cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
+ * invoked on the APs
+ * via i386_init_slave()
+ */
} else {
/* Switch to kernel's page tables (from the Boot PTs) */
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
/* Find our logical cpu number */
cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
+ cpu_desc_load(cpu_datap(cpu));
}
- postcode(VSTART_CPU_DESC_INIT);
- if(is_boot_cpu)
- cpu_desc_init64(cpu_datap(cpu));
- cpu_desc_load64(cpu_datap(cpu));
- postcode(VSTART_CPU_MODE_INIT);
- if (is_boot_cpu)
- cpu_mode_init(current_cpu_datap()); /* cpu_mode_init() will be
- * invoked on the APs
- * via i386_init_slave()
- */
postcode(VSTART_EXIT);
x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
: (uintptr_t) i386_init_slave,
tsc_init();
rtclock_early_init(); /* mach_absolute_time() now functionsl */
- kernel_debug_string_simple("i386_init");
+ kernel_debug_string_early("i386_init");
pstate_trace();
#if CONFIG_MCA
panic_init(); /* Init this in case we need debugger */
/* setup debugging output if one has been chosen */
- kernel_debug_string_simple("PE_init_kprintf");
+ kernel_debug_string_early("PE_init_kprintf");
PE_init_kprintf(FALSE);
- kernel_debug_string_simple("kernel_early_bootstrap");
+ kernel_debug_string_early("kernel_early_bootstrap");
kernel_early_bootstrap();
if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
dgWork.dgFlags = 0;
serialmode = 0;
- if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) {
+ if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
/* We want a serial keyboard and/or console */
kprintf("Serial mode specified: %08X\n", serialmode);
+ int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
+ if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
+ if (force_sync) {
+ serialmode |= SERIALMODE_SYNCDRAIN;
+ kprintf(
+ "WARNING: Forcing uart driver to output synchronously."
+ "printf()s/IOLogs will impact kernel performance.\n"
+ "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
+ }
+ }
}
- if(serialmode & 1) {
+ if (serialmode & SERIALMODE_OUTPUT) {
(void)switch_to_serial_console();
- disableConsoleOutput = FALSE; /* Allow printfs to happen */
+ disableConsoleOutput = FALSE; /* Allow printfs to happen */
}
/* setup console output */
- kernel_debug_string_simple("PE_init_printf");
+ kernel_debug_string_early("PE_init_printf");
PE_init_printf(FALSE);
kprintf("version_variant = %s\n", version_variant);
* VM initialization, after this we're using page tables...
* Thn maximum number of cpus must be set beforehand.
*/
- kernel_debug_string_simple("i386_vm_init");
+ kernel_debug_string_early("i386_vm_init");
i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
/* create the console for verbose or pretty mode */
PE_init_platform(TRUE, kernelBootArgs);
PE_create_console();
- kernel_debug_string_simple("power_management_init");
+ kernel_debug_string_early("power_management_init");
power_management_init();
processor_bootstrap();
thread_bootstrap();
pstate_trace();
- kernel_debug_string_simple("machine_startup");
+ kernel_debug_string_early("machine_startup");
machine_startup();
pstate_trace();
}
assert(!ml_get_interrupts_enabled());
- cpu_mode_init(current_cpu_datap());
+ cpu_syscall_init(current_cpu_datap());
pmap_cpu_init();
#if CONFIG_MCA
do_init_slave(TRUE);
}
+#include <libkern/kernel_mach_header.h>
+
+/* TODO: Evaluate global PTEs for the double-mapped translations */
+
+uint64_t dblmap_base, dblmap_max;
+kernel_segment_command_t *hdescseg;
+
+pt_entry_t *dblmapL3;
+unsigned int dblallocs;
+uint64_t dblmap_dist;
+extern uint64_t idt64_hndl_table0[];
+
+
+void doublemap_init(void) {
+ dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries
+ dblallocs++;
+
+ struct {
+ pt_entry_t entries[PTE_PER_PAGE];
+ } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries
+ dblallocs++;
+
+ dblmapL3[0] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
+ | INTEL_PTE_VALID
+ | INTEL_PTE_WRITE;
+
+ hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB");
+
+ vm_offset_t hdescb = hdescseg->vmaddr;
+ unsigned long hdescsz = hdescseg->vmsize;
+ unsigned long hdescszr = round_page_64(hdescsz);
+ vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr;
+
+ kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text");
+ vm_offset_t thdescb = thdescsect->addr;
+ unsigned long thdescsz = thdescsect->size;
+ unsigned long thdescszr = round_page_64(thdescsz);
+ vm_offset_t thdesce = thdescb + thdescszr;
+
+ assert((hdescb & 0xFFF) == 0);
+ /* Mirror HIB translations into the double-mapped pagetable subtree*/
+ for(int i = 0; hdescc < hdesce; i++) {
+ struct {
+ pt_entry_t entries[PTE_PER_PAGE];
+ } * dblmapL1 = ALLOCPAGES(1);
+ dblallocs++;
+ dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
+ int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE);
+ for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) {
+ uint64_t template = INTEL_PTE_VALID;
+ if ((hdescc >= thdescb) && (hdescc < thdesce)) {
+ /* executable */
+ } else {
+ template |= INTEL_PTE_WRITE | INTEL_PTE_NX ; /* Writeable, NX */
+ }
+ dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
+ hdescc += PAGE_SIZE;
+ }
+ }
+
+ IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
+ dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, dblmapL3, 0, 0);
+ dblmap_max = dblmap_base + hdescszr;
+ /* Calculate the double-map distance, which accounts for the current
+ * KASLR slide
+ */
+
+ dblmap_dist = dblmap_base - hdescb;
+ idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]);
+ idt64_hndl_table0[6] = (uint64_t)(uintptr_t)&kernel_stack_mask;
+
+ extern cpu_data_t cpshadows[], scdatas[];
+ uintptr_t cd1 = (uintptr_t) &cpshadows[0];
+ uintptr_t cd2 = (uintptr_t) &scdatas[0];
+/* Record the displacement from the kernel's per-CPU data pointer, eventually
+ * programmed into GSBASE, to the "shadows" in the doublemapped
+ * region. These are not aliases, but separate physical allocations
+ * containing data required in the doublemapped trampolines.
+*/
+ idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
+
+ DBG("Double map base: 0x%qx\n", dblmap_base);
+ DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]);
+ assert(LDTSZ > LDTSZ_MIN);
+}
+
+vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
+
+#include <i386/pmap_internal.h>
+
+/* Use of this routine is expected to be synchronized by callers
+ * Creates non-executable aliases.
+ */
+vm_offset_t dyn_dblmap(vm_offset_t cva, vm_offset_t sz) {
+ vm_offset_t ava = dblmap_max;
+
+ assert((sz & PAGE_MASK) == 0);
+ assert(cva != 0);
+
+ pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
+ dblmap_max += sz;
+ return (ava - cva);
+}
+/* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect
+ * control to the double-mapped interrupt vectors. The IDTR proper will be
+ * programmed via cpu_desc_load()
+ */
+void idt64_remap(void) {
+ for (int i = 0; i < IDTSZ; i++) {
+ master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
+ }
+}
projects / apple / xnu.git / blobdiff