xnu-4570.1.46.tar.gz

[apple/xnu.git] / osfmk / i386 / i386_init.c

diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c
index 135f7a94272230b12b9ca3b4138cac970593d448..73a30a22b93b387fb33ccc94a0bdef13df3316db 100644 (file)
--- a/osfmk/i386/i386_init.c
+++ b/osfmk/i386/i386_init.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2003-2009 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -54,8 +54,6 @@
  * the rights to redistribute these changes.
  */
 
  * the rights to redistribute these changes.
  */
 

-#include <platforms.h>
-#include <mach_kdb.h>

 
 #include <mach/i386/vm_param.h>
 
 
 #include <mach/i386/vm_param.h>
 


@@ -74,44 +72,48 @@
 #include <kern/xpr.h>
 #include <kern/cpu_data.h>
 #include <kern/processor.h>
 #include <kern/xpr.h>
 #include <kern/cpu_data.h>
 #include <kern/processor.h>

+#include <sys/kdebug.h>

 #include <console/serial_protos.h>
 #include <vm/vm_page.h>
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <console/serial_protos.h>
 #include <vm/vm_page.h>
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>

+#include <machine/pal_routines.h>

 #include <i386/fpu.h>
 #include <i386/pmap.h>
 #include <i386/fpu.h>
 #include <i386/pmap.h>

-#include <i386/ipl.h>

 #include <i386/misc_protos.h>
 #include <i386/cpu_threads.h>
 #include <i386/cpuid.h>
 #include <i386/lapic.h>
 #include <i386/mp.h>
 #include <i386/mp_desc.h>
 #include <i386/misc_protos.h>
 #include <i386/cpu_threads.h>
 #include <i386/cpuid.h>
 #include <i386/lapic.h>
 #include <i386/mp.h>
 #include <i386/mp_desc.h>

+#if CONFIG_MTRR

 #include <i386/mtrr.h>
 #include <i386/mtrr.h>

+#endif

 #include <i386/machine_routines.h>
 #if CONFIG_MCA
 #include <i386/machine_check.h>
 #endif
 #include <i386/machine_routines.h>
 #if CONFIG_MCA
 #include <i386/machine_check.h>
 #endif

+#include <i386/ucode.h>

 #include <i386/postcode.h>
 #include <i386/Diagnostics.h>
 #include <i386/pmCPU.h>
 #include <i386/tsc.h>
 #include <i386/locks.h> /* LcksOpts */
 #include <i386/postcode.h>
 #include <i386/Diagnostics.h>
 #include <i386/pmCPU.h>
 #include <i386/tsc.h>
 #include <i386/locks.h> /* LcksOpts */

-#ifdef __i386__
-#include <i386/cpu_capabilities.h>
-#if    MACH_KDB
-#include <machine/db_machdep.h>
-#endif
+#if DEBUG
+#include <machine/pal_routines.h>

 #endif
 
 #endif
 

+#if MONOTONIC
+#include <kern/monotonic.h>
+#endif /* MONOTONIC */
+
+#include <san/kasan.h>
+

 #if DEBUG
 #define DBG(x...)       kprintf(x)
 #else
 #define DBG(x...)
 #endif
 #if DEBUG
 #define DBG(x...)       kprintf(x)
 #else
 #define DBG(x...)
 #endif

-#if    MACH_KDB
-#include <ddb/db_aout.h>
-#endif /* MACH_KDB */

 
 int                    debug_task;
 
 
 int                    debug_task;
 


@@ -122,20 +124,17 @@ extern const char version[];
 extern const char      version_variant[];
 extern int             nx_enabled;
 
 extern const char      version_variant[];
 extern int             nx_enabled;
 

-extern int             noVMX;  /* if set, rosetta should not emulate altivec */
-
-#ifdef __x86_64__
-extern void            *low_eintstack;
-#endif
-
-extern void serial_init(void);
+/*
+ * Set initial values so that ml_phys_* routines can use the booter's ID mapping
+ * to touch physical space before the kernel's physical aperture exists.
+ */
+uint64_t               physmap_base = 0;
+uint64_t               physmap_max = 4*GB;

 
 

-void                   *KPTphys;
+pd_entry_t             *KPTphys;

 pd_entry_t             *IdlePTD;
 pd_entry_t             *IdlePTD;

-#ifdef __i386__
-pd_entry_t             *IdlePDPT64;
-#endif
-
+pdpt_entry_t           *IdlePDPT;
+pml4_entry_t           *IdlePML4;

 
 char *physfree;
 
 
 char *physfree;
 


@@ -149,9 +148,7 @@ ALLOCPAGES(int npages)
        uintptr_t tmp = (uintptr_t)physfree;
        bzero(physfree, npages * PAGE_SIZE);
        physfree += npages * PAGE_SIZE;
        uintptr_t tmp = (uintptr_t)physfree;
        bzero(physfree, npages * PAGE_SIZE);
        physfree += npages * PAGE_SIZE;

-#ifdef __x86_64__

        tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
        tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;

-#endif

        return (void *)tmp;
 }
 
        return (void *)tmp;
 }
 


@@ -166,17 +163,16 @@ fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
        }
 }
 
        }
 }
 

-extern vm_offset_t first_avail;
+extern pmap_paddr_t first_avail;

 
 

-#ifdef __x86_64__

 int break_kprintf = 0;
 
 uint64_t
 x86_64_pre_sleep(void)
 {
        IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
 int break_kprintf = 0;
 
 uint64_t
 x86_64_pre_sleep(void)
 {
        IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];

-       uint64_t oldcr3 = get_cr3();
-       set_cr3((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
+       uint64_t oldcr3 = get_cr3_raw();
+       set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));

        return oldcr3;
 }
 
        return oldcr3;
 }
 


@@ -184,24 +180,21 @@ void
 x86_64_post_sleep(uint64_t new_cr3)
 {
        IdlePML4[0] = 0;
 x86_64_post_sleep(uint64_t new_cr3)
 {
        IdlePML4[0] = 0;

-       set_cr3((uint32_t) new_cr3);
+       set_cr3_raw((uint32_t) new_cr3);

 }
 
 }
 

-#endif
-
-#ifdef __i386__
-#define ID_MAP_VTOP(x) x
-#endif

 
 
 
 
 
 

-#ifdef __x86_64__

 // Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address
 // NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
 // the PCI hole (which is less 4GB but not more).
 // Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address
 // NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
 // the PCI hole (which is less 4GB but not more).

-#define NPHYSMAP MAX(K64_MAXMEM/GB + 4, 4)
-// Compile-time guard:
-extern int maxphymapsupported[NPHYSMAP <= PTE_PER_PAGE ? 1 : -1];
+
+/* Compile-time guard: NPHYSMAP is capped to 256GiB, accounting for
+ * randomisation
+ */
+extern int maxphymapsupported[NPHYSMAP <= (PTE_PER_PAGE/2) ? 1 : -1];
+

 static void
 physmap_init(void)
 {
 static void
 physmap_init(void)
 {


@@ -210,27 +203,78 @@ physmap_init(void)
                pt_entry_t entries[PTE_PER_PAGE];
        } * physmapL2 = ALLOCPAGES(NPHYSMAP);
 
                pt_entry_t entries[PTE_PER_PAGE];
        } * physmapL2 = ALLOCPAGES(NPHYSMAP);
 

-       uintptr_t i;
-       for(i=0;i<NPHYSMAP;i++) {
-               physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(&physmapL2[i]))
+       uint64_t i;
+       uint8_t phys_random_L3 = early_random() & 0xFF;
+
+       /* We assume NX support. Mark all levels of the PHYSMAP NX
+        * to avoid granting executability via a single bit flip.
+        */
+#if DEVELOPMENT || DEBUG
+       uint32_t reg[4];
+       do_cpuid(0x80000000, reg);
+       if (reg[eax] >= 0x80000001) {
+               do_cpuid(0x80000001, reg);
+               assert(reg[edx] & CPUID_EXTFEATURE_XD);
+       }
+#endif /* DEVELOPMENT || DEBUG */
+
+       for(i = 0; i < NPHYSMAP; i++) {
+               physmapL3[i + phys_random_L3] =
+                               ((uintptr_t)ID_MAP_VTOP(&physmapL2[i]))

                                | INTEL_PTE_VALID
                                | INTEL_PTE_VALID

+                               | INTEL_PTE_NX

                                | INTEL_PTE_WRITE;
                                | INTEL_PTE_WRITE;

-               uintptr_t j;
-               for(j=0;j<PTE_PER_PAGE;j++) {
-                       physmapL2[i].entries[j] = (((i*PTE_PER_PAGE+j)<<PDSHIFT)
+
+               uint64_t j;
+               for(j = 0; j < PTE_PER_PAGE; j++) {
+                       physmapL2[i].entries[j] =
+                           ((i * PTE_PER_PAGE + j) << PDSHIFT)

                                                        | INTEL_PTE_PS
                                                        | INTEL_PTE_VALID
                                                        | INTEL_PTE_PS
                                                        | INTEL_PTE_VALID

-                                                       | INTEL_PTE_WRITE);
+                                                       | INTEL_PTE_NX
+                                                       | INTEL_PTE_WRITE;

                }
        }
 
                }
        }
 

-       IdlePML4[KERNEL_PHYSMAP_INDEX] = ((uintptr_t)ID_MAP_VTOP(physmapL3))
-                                               | INTEL_PTE_VALID
-                                               | INTEL_PTE_WRITE;
-       DBG("physical map idlepml4[%d]: 0x%llx\n",
-               KERNEL_PHYSMAP_INDEX, IdlePML4[KERNEL_PHYSMAP_INDEX]);
+       IdlePML4[KERNEL_PHYSMAP_PML4_INDEX] =
+                                       ((uintptr_t)ID_MAP_VTOP(physmapL3))
+                                       | INTEL_PTE_VALID
+                                       | INTEL_PTE_NX
+                                       | INTEL_PTE_WRITE;
+
+       physmap_base = KVADDR(KERNEL_PHYSMAP_PML4_INDEX, phys_random_L3, 0, 0);
+       physmap_max = physmap_base + NPHYSMAP * GB;
+       DBG("Physical address map base: 0x%qx\n", physmap_base);
+       DBG("Physical map idlepml4[%d]: 0x%llx\n",
+               KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
+}
+
+static void
+descriptor_alias_init()
+{
+       vm_offset_t     master_gdt_phys;
+       vm_offset_t     master_gdt_alias_phys;
+       vm_offset_t     master_idt_phys;
+       vm_offset_t     master_idt_alias_phys;
+
+       assert(((vm_offset_t)master_gdt & PAGE_MASK) == 0);
+       assert(((vm_offset_t)master_idt64 & PAGE_MASK) == 0);
+
+       master_gdt_phys       = (vm_offset_t) ID_MAP_VTOP(master_gdt);
+       master_idt_phys       = (vm_offset_t) ID_MAP_VTOP(master_idt64);
+       master_gdt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_GDT_ALIAS);
+       master_idt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_IDT_ALIAS);
+       
+       DBG("master_gdt_phys:       %p\n", (void *) master_gdt_phys);
+       DBG("master_idt_phys:       %p\n", (void *) master_idt_phys);
+       DBG("master_gdt_alias_phys: %p\n", (void *) master_gdt_alias_phys);
+       DBG("master_idt_alias_phys: %p\n", (void *) master_idt_alias_phys);
+
+       KPTphys[atop_kernel(master_gdt_alias_phys)] = master_gdt_phys |
+               INTEL_PTE_VALID | INTEL_PTE_NX | INTEL_PTE_WRITE;
+       KPTphys[atop_kernel(master_idt_alias_phys)] = master_idt_phys |
+               INTEL_PTE_VALID | INTEL_PTE_NX; /* read-only */

 }
 }

-#endif

 
 static void
 Idle_PTs_init(void)
 
 static void
 Idle_PTs_init(void)


@@ -238,36 +282,80 @@ Idle_PTs_init(void)
        /* Allocate the "idle" kernel page tables: */
        KPTphys  = ALLOCPAGES(NKPT);            /* level 1 */
        IdlePTD  = ALLOCPAGES(NPGPTD);          /* level 2 */
        /* Allocate the "idle" kernel page tables: */
        KPTphys  = ALLOCPAGES(NKPT);            /* level 1 */
        IdlePTD  = ALLOCPAGES(NPGPTD);          /* level 2 */

+       IdlePDPT = ALLOCPAGES(1);               /* level 3 */
+       IdlePML4 = ALLOCPAGES(1);               /* level 4 */

 
 

-#ifdef __x86_64__
-       physmap_init();
-#else
-       IdlePDPT64 = ALLOCPAGES(1);
-
-       // Recursive mapping of PTEs
-       fillkpt(IdlePTD, INTEL_PTE_WRITE, (uintptr_t)IdlePTD, PTDPTDI, NPGPTD);
-       // commpage
-       fillkpt(IdlePTD, INTEL_PTE_WRITE|INTEL_PTE_USER, (uintptr_t)ALLOCPAGES(1), _COMM_PAGE32_BASE_ADDRESS >> PDESHIFT,1);
-#endif

        // Fill the lowest level with everything up to physfree
        fillkpt(KPTphys,
        // Fill the lowest level with everything up to physfree
        fillkpt(KPTphys,

-                       INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
+               INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));

 
 

-       // Rewrite the 2nd-lowest level  to point to pages of KPTphys.
-       // This was previously filled statically by idle_pt.c, and thus
-       // must be done after the KPTphys fill since IdlePTD is in use
+       /* IdlePTD */

        fillkpt(IdlePTD,
        fillkpt(IdlePTD,

-                       INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
+               INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);

 
        // IdlePDPT entries
 
        // IdlePDPT entries

-#ifdef __i386__
-       fillkpt(IdlePDPT, 0, (uintptr_t)IdlePTD, 0, NPGPTD);
-#else
-       fillkpt(IdlePDPT, INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
-#endif
+       fillkpt(IdlePDPT,
+               INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
+
+       // IdlePML4 single entry for kernel space.
+       fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
+               INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
+       
+       postcode(VSTART_PHYSMAP_INIT);
+
+       physmap_init();
+
+       postcode(VSTART_DESC_ALIAS_INIT);
+
+       descriptor_alias_init();
+
+       postcode(VSTART_SET_CR3);
+
+       // Switch to the page tables..
+       set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));

 
 

-       // Flush the TLB now we're done rewriting the page tables..
-       set_cr3(get_cr3());
+}
+
+extern void vstart_trap_handler;
+
+#define BOOT_TRAP_VECTOR(t)                            \
+       [t] = {                                         \
+               (uintptr_t) &vstart_trap_handler,       \
+               KERNEL64_CS,                            \
+               0,                                      \
+               ACC_P|ACC_PL_K|ACC_INTR_GATE,           \
+               0                                       \
+       },
+
+/* Recursive macro to iterate 0..31 */
+#define L0(x,n)         x(n)
+#define L1(x,n)         L0(x,n-1)     L0(x,n)
+#define L2(x,n)  L1(x,n-2)     L1(x,n)
+#define L3(x,n)  L2(x,n-4)     L2(x,n)
+#define L4(x,n)  L3(x,n-8)     L3(x,n)
+#define L5(x,n)  L4(x,n-16)    L4(x,n)
+#define FOR_0_TO_31(x) L5(x,31)
+
+/*
+ * Bootstrap IDT. Active only during early startup.
+ * Only the trap vectors are defined since interrupts are masked.
+ * All traps point to a common handler.
+ */
+struct fake_descriptor64 master_boot_idt64[IDTSZ]
+       __attribute__((section("__HIB,__desc")))
+       __attribute__((aligned(PAGE_SIZE))) = {
+       FOR_0_TO_31(BOOT_TRAP_VECTOR)
+};
+
+static void
+vstart_idt_init(void)
+{
+       x86_64_desc_register_t  vstart_idt = {
+                                       sizeof(master_boot_idt64),
+                                       master_boot_idt64 };
+       
+       fix_desc64(master_boot_idt64, 32);
+       lidt((void *)&vstart_idt);

 }
 
 /*
 }
 
 /*


@@ -285,24 +373,32 @@ Idle_PTs_init(void)
  * Non-bootstrap processors are called with argument boot_args_start NULL.
  * These processors switch immediately to the existing kernel page tables.
  */
  * Non-bootstrap processors are called with argument boot_args_start NULL.
  * These processors switch immediately to the existing kernel page tables.
  */

+__attribute__((noreturn))

 void
 vstart(vm_offset_t boot_args_start)
 {
        boolean_t       is_boot_cpu = !(boot_args_start == 0);
 void
 vstart(vm_offset_t boot_args_start)
 {
        boolean_t       is_boot_cpu = !(boot_args_start == 0);

-       int             cpu;
-       uint32_t lphysfree;
+       int             cpu = 0;
+       uint32_t        lphysfree;

 
        postcode(VSTART_ENTRY);
 
        if (is_boot_cpu) {
 
        postcode(VSTART_ENTRY);
 
        if (is_boot_cpu) {

+               /*
+                * Set-up temporary trap handlers during page-table set-up.
+                */
+               vstart_idt_init();
+               postcode(VSTART_IDT_INIT);
+

                /*
                 * Get startup parameters.
                 */
                kernelBootArgs = (boot_args *)boot_args_start;
                lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
                physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1));
                /*
                 * Get startup parameters.
                 */
                kernelBootArgs = (boot_args *)boot_args_start;
                lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
                physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1));

-#if DEBUG
-               serial_init();
+
+#if DEVELOPMENT || DEBUG
+               pal_serial_init();

 #endif
                DBG("revision      0x%x\n", kernelBootArgs->Revision);
                DBG("version       0x%x\n", kernelBootArgs->Version);
 #endif
                DBG("revision      0x%x\n", kernelBootArgs->Revision);
                DBG("version       0x%x\n", kernelBootArgs->Version);


@@ -316,71 +412,70 @@ vstart(vm_offset_t boot_args_start)
                        kernelBootArgs, 
                        &kernelBootArgs->ksize,
                        &kernelBootArgs->kaddr);
                        kernelBootArgs, 
                        &kernelBootArgs->ksize,
                        &kernelBootArgs->kaddr);

+               DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize);

 
 

-               postcode(PSTART_PAGE_TABLES);
+               /*
+                * Setup boot args given the physical start address.
+                * Note: PE_init_platform needs to be called before Idle_PTs_init
+                * because access to the DeviceTree is required to read the
+                * random seed before generating a random physical map slide.
+                */
+               kernelBootArgs = (boot_args *)
+                   ml_static_ptovirt(boot_args_start);
+               DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
+                   (unsigned long)boot_args_start, kernelBootArgs);
+
+#if KASAN
+               kasan_reserve_memory(kernelBootArgs);
+#endif
+
+               PE_init_platform(FALSE, kernelBootArgs);
+               postcode(PE_INIT_PLATFORM_D);

 
                Idle_PTs_init();
 
                Idle_PTs_init();

+               postcode(VSTART_IDLE_PTS_INIT);
+
+#if KASAN
+               /* Init kasan and map whatever was stolen from physfree */
+               kasan_init();
+               kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
+#endif
+
+#if MONOTONIC
+               mt_init();
+#endif /* MONOTONIC */

 
                first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
 
 
                first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
 

-               cpu = 0;
+               cpu_data_alloc(TRUE);
+
+               cpu_desc_init(cpu_datap(0));
+               postcode(VSTART_CPU_DESC_INIT);
+               cpu_desc_load(cpu_datap(0));
+
+               postcode(VSTART_CPU_MODE_INIT);
+               cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
+                                                * invoked on the APs
+                                                * via i386_init_slave()
+                                                */

        } else {
        } else {

+               /* Switch to kernel's page tables (from the Boot PTs) */
+               set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));

                /* Find our logical cpu number */
                cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
                /* Find our logical cpu number */
                cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];

+               DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
+               cpu_desc_load(cpu_datap(cpu));

        }
 
        }
 

-       if(is_boot_cpu) cpu_data_alloc(TRUE);
-#ifdef __x86_64__
-       if(is_boot_cpu)
-               cpu_desc_init64(cpu_datap(cpu));
-       cpu_desc_load64(cpu_datap(cpu));
-#else
-       if(is_boot_cpu)
-               cpu_desc_init(cpu_datap(cpu));
-       cpu_desc_load(cpu_datap(cpu));
-#endif
-       cpu_mode_init(current_cpu_datap());
-
-       /* enable NX/XD */
-       if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD)
-        wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_NXE);
-       DBG("vstart() NX/XD enabled\n");
-
-
-#ifdef __x86_64__
-       /* Done with identity mapping */
-       IdlePML4[0] = 0;
-#endif
-

        postcode(VSTART_EXIT);
        postcode(VSTART_EXIT);

-#ifdef __i386__
-       if (is_boot_cpu)
-               i386_init(boot_args_start);
-       else
-               i386_init_slave();
-       /*NOTREACHED*/
-#else
-       /* We need to switch to a new per-cpu stack, but we must do this atomically with
-        * the call to ensure the compiler doesn't assume anything about the stack before
-        * e.g. tail-call optimisations
-        */
-       if (is_boot_cpu)
-       {
-               asm volatile(
-                               "mov %1, %%rdi;"
-                               "mov %0, %%rsp;"
-                               "call _i386_init;"      : : "r" 
-                               (cpu_datap(cpu)->cpu_int_stack_top), "r" (boot_args_start));
-       }
-       else
-       {
-               asm volatile(
-                               "mov %0, %%rsp;"
-                               "call _i386_init_slave;"        : : "r" 
-                               (cpu_datap(cpu)->cpu_int_stack_top));
-       }
-       /*NOTREACHED*/
-#endif
+       x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
+                                    : (uintptr_t) i386_init_slave,
+                        cpu_datap(cpu)->cpu_int_stack_top);
+}
+
+void
+pstate_trace(void)
+{

 }
 
 /*
 }
 
 /*


@@ -388,63 +483,68 @@ vstart(vm_offset_t boot_args_start)
  *     set up.
  */
 void
  *     set up.
  */
 void

-i386_init(vm_offset_t boot_args_start)
+i386_init(void)

 {
        unsigned int    maxmem;
        uint64_t        maxmemtouse;
        unsigned int    cpus = 0;
        boolean_t       fidn;
 {
        unsigned int    maxmem;
        uint64_t        maxmemtouse;
        unsigned int    cpus = 0;
        boolean_t       fidn;

-#ifdef __i386__
-       boolean_t       legacy_mode;
-#endif

        boolean_t       IA32e = TRUE;
 
        postcode(I386_INIT_ENTRY);
 
        boolean_t       IA32e = TRUE;
 
        postcode(I386_INIT_ENTRY);
 

+       pal_i386_init();
+       tsc_init();
+       rtclock_early_init();   /* mach_absolute_time() now functionsl */
+
+       kernel_debug_string_early("i386_init");
+       pstate_trace();
+

 #if CONFIG_MCA
        /* Initialize machine-check handling */
        mca_cpu_init();
 #endif
 
 #if CONFIG_MCA
        /* Initialize machine-check handling */
        mca_cpu_init();
 #endif
 

-       /*
-        * Setup boot args given the physical start address.
-        */
-       kernelBootArgs = (boot_args *)
-               ml_static_ptovirt(boot_args_start);
-       DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
-               (unsigned long)boot_args_start, kernelBootArgs);
-

        master_cpu = 0;
        cpu_init();
 
        postcode(CPU_INIT_D);
 
        master_cpu = 0;
        cpu_init();
 
        postcode(CPU_INIT_D);
 

-
-       PE_init_platform(FALSE, kernelBootArgs);
-       postcode(PE_INIT_PLATFORM_D);
-
-

        printf_init();                  /* Init this in case we need debugger */
        panic_init();                   /* Init this in case we need debugger */
 
        printf_init();                  /* Init this in case we need debugger */
        panic_init();                   /* Init this in case we need debugger */
 

-

        /* setup debugging output if one has been chosen */
        /* setup debugging output if one has been chosen */

+       kernel_debug_string_early("PE_init_kprintf");

        PE_init_kprintf(FALSE);
 
        PE_init_kprintf(FALSE);
 

+       kernel_debug_string_early("kernel_early_bootstrap");
+       kernel_early_bootstrap();
+

        if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
                dgWork.dgFlags = 0;
 
        serialmode = 0;
        if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
                dgWork.dgFlags = 0;
 
        serialmode = 0;

-       if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) {
+       if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {

                /* We want a serial keyboard and/or console */
                kprintf("Serial mode specified: %08X\n", serialmode);
                /* We want a serial keyboard and/or console */
                kprintf("Serial mode specified: %08X\n", serialmode);

+               int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
+               if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
+                       if (force_sync) {
+                               serialmode |= SERIALMODE_SYNCDRAIN;
+                               kprintf(
+                                   "WARNING: Forcing uart driver to output synchronously."
+                                   "printf()s/IOLogs will impact kernel performance.\n"
+                                   "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
+                       }
+               }

        }
        }

-       if(serialmode & 1) {
+       if (serialmode & SERIALMODE_OUTPUT) {

                (void)switch_to_serial_console();
                (void)switch_to_serial_console();

-               disableConsoleOutput = FALSE;   /* Allow printfs to happen */
+               disableConsoleOutput = FALSE; /* Allow printfs to happen */

        }
 
        /* setup console output */
        }
 
        /* setup console output */

+       kernel_debug_string_early("PE_init_printf");

        PE_init_printf(FALSE);
 
        kprintf("version_variant = %s\n", version_variant);
        PE_init_printf(FALSE);
 
        kprintf("version_variant = %s\n", version_variant);


@@ -460,62 +560,49 @@ i386_init(vm_offset_t boot_args_start)
                         max_ncpus = cpus;
        }
 
                         max_ncpus = cpus;
        }
 

-

        /*
         * debug support for > 4G systems
         */
        /*
         * debug support for > 4G systems
         */

-       if (!PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode)))
-               vm_himemory_mode = 0;
+       PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode));
+       if (vm_himemory_mode != 0)
+               kprintf("himemory_mode: %d\n", vm_himemory_mode);

 
        if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof (fidn)))
                force_immediate_debugger_NMI = FALSE;
        else
                force_immediate_debugger_NMI = fidn;
 
        if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof (fidn)))
                force_immediate_debugger_NMI = FALSE;
        else
                force_immediate_debugger_NMI = fidn;

-#ifdef __i386__
-       /*
-        * At this point we check whether we are a 64-bit processor
-        * and that we're not restricted to legacy mode, 32-bit operation.
-        */
-       if (cpuid_extfeatures() & CPUID_EXTFEATURE_EM64T) {
-               kprintf("EM64T supported");
-               if (PE_parse_boot_argn("-legacy", &legacy_mode, sizeof (legacy_mode))) {
-                       kprintf(" but legacy mode forced\n");
-                       IA32e = FALSE;
-               } else {
-                       kprintf(" and will be enabled\n");
-               }
-       } else
-               IA32e = FALSE;
+
+#if DEBUG
+       nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);

 #endif
 #endif

+       PE_parse_boot_argn("urgency_notification_abstime",
+           &urgency_notification_assert_abstime_threshold,
+           sizeof(urgency_notification_assert_abstime_threshold));

 
        if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
                nx_enabled = 0;
 
 
        if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
                nx_enabled = 0;
 

-       /* Obtain "lcks" options:this currently controls lock statistics */
-       if (!PE_parse_boot_argn("lcks", &LcksOpts, sizeof (LcksOpts)))
-               LcksOpts = 0;
-

        /*   
         * VM initialization, after this we're using page tables...
        /*   
         * VM initialization, after this we're using page tables...

-        * The maximum number of cpus must be set beforehand.
+        * Thn maximum number of cpus must be set beforehand.

         */
         */

+       kernel_debug_string_early("i386_vm_init");

        i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
 
        i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
 

-       if ( ! PE_parse_boot_argn("novmx", &noVMX, sizeof (noVMX)))
-               noVMX = 0;      /* OK to support Altivec in rosetta? */
-

        /* create the console for verbose or pretty mode */
        /* Note: doing this prior to tsc_init() allows for graceful panic! */
        PE_init_platform(TRUE, kernelBootArgs);
        PE_create_console();
 
        /* create the console for verbose or pretty mode */
        /* Note: doing this prior to tsc_init() allows for graceful panic! */
        PE_init_platform(TRUE, kernelBootArgs);
        PE_create_console();
 

-       tsc_init();
+       kernel_debug_string_early("power_management_init");

        power_management_init();
        power_management_init();

-

        processor_bootstrap();
        thread_bootstrap();
 
        processor_bootstrap();
        thread_bootstrap();
 

+       pstate_trace();
+       kernel_debug_string_early("machine_startup");

        machine_startup();
        machine_startup();

+       pstate_trace();

 }
 
 static void
 }
 
 static void


@@ -534,42 +621,41 @@ do_init_slave(boolean_t fast_restart)
   
                assert(!ml_get_interrupts_enabled());
   
   
                assert(!ml_get_interrupts_enabled());
   

-               cpu_mode_init(current_cpu_datap());
+               cpu_syscall_init(current_cpu_datap());
+               pmap_cpu_init();

   
 #if CONFIG_MCA
                mca_cpu_init();
 #endif
   
   
 #if CONFIG_MCA
                mca_cpu_init();
 #endif
   

+               LAPIC_INIT();

                lapic_configure();
                LAPIC_DUMP();
                LAPIC_CPU_MAP_DUMP();
   
                init_fpu();
   
                lapic_configure();
                LAPIC_DUMP();
                LAPIC_CPU_MAP_DUMP();
   
                init_fpu();
   

+#if CONFIG_MTRR

                mtrr_update_cpu();
                mtrr_update_cpu();

+#endif
+               /* update CPU microcode */
+               ucode_update_wake();

        } else
            init_param = FAST_SLAVE_INIT;
 
 #if CONFIG_VMX
        /* resume VT operation */
        } else
            init_param = FAST_SLAVE_INIT;
 
 #if CONFIG_VMX
        /* resume VT operation */

-       vmx_resume();
+       vmx_resume(FALSE);

 #endif
 
 #endif
 

+#if CONFIG_MTRR

        if (!fast_restart)
            pat_init();
        if (!fast_restart)
            pat_init();

-
-       cpu_thread_init();      /* not strictly necessary */
-
-#ifdef __x86_64__
-       /* Re-zero the identity-map for the idle PT's. This MUST be done before 
-        * cpu_running is set so that other slaves can set up their own
-        * identity-map */
-       if (!fast_restart)
-           IdlePML4[0] = 0;

 #endif
 
 #endif
 

-       cpu_init();     /* Sets cpu_running which starter cpu waits for */ 
+       cpu_thread_init();      /* not strictly necessary */

 
 

+       cpu_init();     /* Sets cpu_running which starter cpu waits for */

        slave_main(init_param);
   
        panic("do_init_slave() returned from slave_main()");
        slave_main(init_param);
   
        panic("do_init_slave() returned from slave_main()");