xnu-4570.1.46.tar.gz

[apple/xnu.git] / osfmk / i386 / i386_init.c

diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c
index dc528cce12c9ad96a4231e83efc03b9b859f6f58..73a30a22b93b387fb33ccc94a0bdef13df3316db 100644 (file)
--- a/osfmk/i386/i386_init.c
+++ b/osfmk/i386/i386_init.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -102,6 +102,13 @@
 #if DEBUG
 #include <machine/pal_routines.h>
 #endif
 #if DEBUG
 #include <machine/pal_routines.h>
 #endif

+
+#if MONOTONIC
+#include <kern/monotonic.h>
+#endif /* MONOTONIC */
+
+#include <san/kasan.h>
+

 #if DEBUG
 #define DBG(x...)       kprintf(x)
 #else
 #if DEBUG
 #define DBG(x...)       kprintf(x)
 #else


@@ -117,7 +124,12 @@ extern const char  version[];
 extern const char      version_variant[];
 extern int             nx_enabled;
 
 extern const char      version_variant[];
 extern int             nx_enabled;
 

-uint64_t               physmap_base, physmap_max;
+/*
+ * Set initial values so that ml_phys_* routines can use the booter's ID mapping
+ * to touch physical space before the kernel's physical aperture exists.
+ */
+uint64_t               physmap_base = 0;
+uint64_t               physmap_max = 4*GB;

 
 pd_entry_t             *KPTphys;
 pd_entry_t             *IdlePTD;
 
 pd_entry_t             *KPTphys;
 pd_entry_t             *IdlePTD;


@@ -304,6 +316,47 @@ Idle_PTs_init(void)
 
 }
 
 
 }
 

+extern void vstart_trap_handler;
+
+#define BOOT_TRAP_VECTOR(t)                            \
+       [t] = {                                         \
+               (uintptr_t) &vstart_trap_handler,       \
+               KERNEL64_CS,                            \
+               0,                                      \
+               ACC_P|ACC_PL_K|ACC_INTR_GATE,           \
+               0                                       \
+       },
+
+/* Recursive macro to iterate 0..31 */
+#define L0(x,n)         x(n)
+#define L1(x,n)         L0(x,n-1)     L0(x,n)
+#define L2(x,n)  L1(x,n-2)     L1(x,n)
+#define L3(x,n)  L2(x,n-4)     L2(x,n)
+#define L4(x,n)  L3(x,n-8)     L3(x,n)
+#define L5(x,n)  L4(x,n-16)    L4(x,n)
+#define FOR_0_TO_31(x) L5(x,31)
+
+/*
+ * Bootstrap IDT. Active only during early startup.
+ * Only the trap vectors are defined since interrupts are masked.
+ * All traps point to a common handler.
+ */
+struct fake_descriptor64 master_boot_idt64[IDTSZ]
+       __attribute__((section("__HIB,__desc")))
+       __attribute__((aligned(PAGE_SIZE))) = {
+       FOR_0_TO_31(BOOT_TRAP_VECTOR)
+};
+
+static void
+vstart_idt_init(void)
+{
+       x86_64_desc_register_t  vstart_idt = {
+                                       sizeof(master_boot_idt64),
+                                       master_boot_idt64 };
+       
+       fix_desc64(master_boot_idt64, 32);
+       lidt((void *)&vstart_idt);
+}

 
 /*
  * vstart() is called in the natural mode (64bit for K64, 32 for K32)
 
 /*
  * vstart() is called in the natural mode (64bit for K64, 32 for K32)


@@ -320,16 +373,23 @@ Idle_PTs_init(void)
  * Non-bootstrap processors are called with argument boot_args_start NULL.
  * These processors switch immediately to the existing kernel page tables.
  */
  * Non-bootstrap processors are called with argument boot_args_start NULL.
  * These processors switch immediately to the existing kernel page tables.
  */

+__attribute__((noreturn))

 void
 vstart(vm_offset_t boot_args_start)
 {
        boolean_t       is_boot_cpu = !(boot_args_start == 0);
 void
 vstart(vm_offset_t boot_args_start)
 {
        boolean_t       is_boot_cpu = !(boot_args_start == 0);

-       int             cpu;
+       int             cpu = 0;

        uint32_t        lphysfree;
 
        postcode(VSTART_ENTRY);
 
        if (is_boot_cpu) {
        uint32_t        lphysfree;
 
        postcode(VSTART_ENTRY);
 
        if (is_boot_cpu) {

+               /*
+                * Set-up temporary trap handlers during page-table set-up.
+                */
+               vstart_idt_init();
+               postcode(VSTART_IDT_INIT);
+

                /*
                 * Get startup parameters.
                 */
                /*
                 * Get startup parameters.
                 */


@@ -364,34 +424,49 @@ vstart(vm_offset_t boot_args_start)
                    ml_static_ptovirt(boot_args_start);
                DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
                    (unsigned long)boot_args_start, kernelBootArgs);
                    ml_static_ptovirt(boot_args_start);
                DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
                    (unsigned long)boot_args_start, kernelBootArgs);

+
+#if KASAN
+               kasan_reserve_memory(kernelBootArgs);
+#endif
+

                PE_init_platform(FALSE, kernelBootArgs);
                postcode(PE_INIT_PLATFORM_D);
 
                Idle_PTs_init();
                postcode(VSTART_IDLE_PTS_INIT);
 
                PE_init_platform(FALSE, kernelBootArgs);
                postcode(PE_INIT_PLATFORM_D);
 
                Idle_PTs_init();
                postcode(VSTART_IDLE_PTS_INIT);
 

+#if KASAN
+               /* Init kasan and map whatever was stolen from physfree */
+               kasan_init();
+               kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
+#endif
+
+#if MONOTONIC
+               mt_init();
+#endif /* MONOTONIC */
+

                first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
 
                first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
 

-               cpu = 0;

                cpu_data_alloc(TRUE);
                cpu_data_alloc(TRUE);

+
+               cpu_desc_init(cpu_datap(0));
+               postcode(VSTART_CPU_DESC_INIT);
+               cpu_desc_load(cpu_datap(0));
+
+               postcode(VSTART_CPU_MODE_INIT);
+               cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
+                                                * invoked on the APs
+                                                * via i386_init_slave()
+                                                */

        } else {
                /* Switch to kernel's page tables (from the Boot PTs) */
                set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
                /* Find our logical cpu number */
                cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
                DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
        } else {
                /* Switch to kernel's page tables (from the Boot PTs) */
                set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
                /* Find our logical cpu number */
                cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
                DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));

+               cpu_desc_load(cpu_datap(cpu));

        }
 
        }
 

-       postcode(VSTART_CPU_DESC_INIT);
-       if(is_boot_cpu)
-               cpu_desc_init64(cpu_datap(cpu));
-       cpu_desc_load64(cpu_datap(cpu));
-       postcode(VSTART_CPU_MODE_INIT);
-       if (is_boot_cpu)
-               cpu_mode_init(current_cpu_datap()); /* cpu_mode_init() will be
-                                                    * invoked on the APs
-                                                    * via i386_init_slave()
-                                                    */

        postcode(VSTART_EXIT);
        x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
                                     : (uintptr_t) i386_init_slave,
        postcode(VSTART_EXIT);
        x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
                                     : (uintptr_t) i386_init_slave,


@@ -422,7 +497,7 @@ i386_init(void)
        tsc_init();
        rtclock_early_init();   /* mach_absolute_time() now functionsl */
 
        tsc_init();
        rtclock_early_init();   /* mach_absolute_time() now functionsl */
 

-       kernel_debug_string_simple("i386_init");
+       kernel_debug_string_early("i386_init");

        pstate_trace();
 
 #if CONFIG_MCA
        pstate_trace();
 
 #if CONFIG_MCA


@@ -439,27 +514,37 @@ i386_init(void)
        panic_init();                   /* Init this in case we need debugger */
 
        /* setup debugging output if one has been chosen */
        panic_init();                   /* Init this in case we need debugger */
 
        /* setup debugging output if one has been chosen */

-       kernel_debug_string_simple("PE_init_kprintf");
+       kernel_debug_string_early("PE_init_kprintf");

        PE_init_kprintf(FALSE);
 
        PE_init_kprintf(FALSE);
 

-       kernel_debug_string_simple("kernel_early_bootstrap");
+       kernel_debug_string_early("kernel_early_bootstrap");

        kernel_early_bootstrap();
 
        if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
                dgWork.dgFlags = 0;
 
        serialmode = 0;
        kernel_early_bootstrap();
 
        if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
                dgWork.dgFlags = 0;
 
        serialmode = 0;

-       if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) {
+       if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {

                /* We want a serial keyboard and/or console */
                kprintf("Serial mode specified: %08X\n", serialmode);
                /* We want a serial keyboard and/or console */
                kprintf("Serial mode specified: %08X\n", serialmode);

+               int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
+               if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
+                       if (force_sync) {
+                               serialmode |= SERIALMODE_SYNCDRAIN;
+                               kprintf(
+                                   "WARNING: Forcing uart driver to output synchronously."
+                                   "printf()s/IOLogs will impact kernel performance.\n"
+                                   "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
+                       }
+               }

        }
        }

-       if(serialmode & 1) {
+       if (serialmode & SERIALMODE_OUTPUT) {

                (void)switch_to_serial_console();
                (void)switch_to_serial_console();

-               disableConsoleOutput = FALSE;   /* Allow printfs to happen */
+               disableConsoleOutput = FALSE; /* Allow printfs to happen */

        }
 
        /* setup console output */
        }
 
        /* setup console output */

-       kernel_debug_string_simple("PE_init_printf");
+       kernel_debug_string_early("PE_init_printf");

        PE_init_printf(FALSE);
 
        kprintf("version_variant = %s\n", version_variant);
        PE_init_printf(FALSE);
 
        kprintf("version_variant = %s\n", version_variant);


@@ -501,7 +586,7 @@ i386_init(void)
         * VM initialization, after this we're using page tables...
         * Thn maximum number of cpus must be set beforehand.
         */
         * VM initialization, after this we're using page tables...
         * Thn maximum number of cpus must be set beforehand.
         */

-       kernel_debug_string_simple("i386_vm_init");
+       kernel_debug_string_early("i386_vm_init");

        i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
 
        /* create the console for verbose or pretty mode */
        i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
 
        /* create the console for verbose or pretty mode */


@@ -509,13 +594,13 @@ i386_init(void)
        PE_init_platform(TRUE, kernelBootArgs);
        PE_create_console();
 
        PE_init_platform(TRUE, kernelBootArgs);
        PE_create_console();
 

-       kernel_debug_string_simple("power_management_init");
+       kernel_debug_string_early("power_management_init");

        power_management_init();
        processor_bootstrap();
        thread_bootstrap();
 
        pstate_trace();
        power_management_init();
        processor_bootstrap();
        thread_bootstrap();
 
        pstate_trace();

-       kernel_debug_string_simple("machine_startup");
+       kernel_debug_string_early("machine_startup");

        machine_startup();
        pstate_trace();
 }
        machine_startup();
        pstate_trace();
 }


@@ -536,7 +621,7 @@ do_init_slave(boolean_t fast_restart)
   
                assert(!ml_get_interrupts_enabled());
   
   
                assert(!ml_get_interrupts_enabled());
   

-               cpu_mode_init(current_cpu_datap());
+               cpu_syscall_init(current_cpu_datap());

                pmap_cpu_init();
   
 #if CONFIG_MCA
                pmap_cpu_init();
   
 #if CONFIG_MCA


@@ -560,7 +645,7 @@ do_init_slave(boolean_t fast_restart)
 
 #if CONFIG_VMX
        /* resume VT operation */
 
 #if CONFIG_VMX
        /* resume VT operation */

-       vmx_resume();
+       vmx_resume(FALSE);

 #endif
 
 #if CONFIG_MTRR
 #endif
 
 #if CONFIG_MTRR