/*
- * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
+ *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
- *
+ *
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
+ *
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
-/*
+/*
* Mach Operating System
* Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
* All Rights Reserved.
- *
+ *
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
- *
+ *
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- *
+ *
* Carnegie Mellon requests users of this software to return to
- *
+ *
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
- *
+ *
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
#include <mach/i386/vm_param.h>
#include <string.h>
+#include <stdint.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <mach/machine.h>
#include <san/kasan.h>
#if DEBUG
-#define DBG(x...) kprintf(x)
+#define DBG(x ...) kprintf(x)
#else
-#define DBG(x...)
+#define DBG(x ...)
#endif
-int debug_task;
+int debug_task;
-static boot_args *kernelBootArgs;
+int early_boot = 1;
-extern int disableConsoleOutput;
-extern const char version[];
-extern const char version_variant[];
-extern int nx_enabled;
+static boot_args *kernelBootArgs;
+
+extern int disableConsoleOutput;
+extern const char version[];
+extern const char version_variant[];
+extern int nx_enabled;
/*
* Set initial values so that ml_phys_* routines can use the booter's ID mapping
* to touch physical space before the kernel's physical aperture exists.
*/
-uint64_t physmap_base = 0;
-uint64_t physmap_max = 4*GB;
+uint64_t physmap_base = 0;
+uint64_t physmap_max = 4 * GB;
+
+pd_entry_t *KPTphys;
+pd_entry_t *IdlePTD;
+pdpt_entry_t *IdlePDPT;
+pml4_entry_t *IdlePML4;
+
+int kernPhysPML4Index;
+int kernPhysPML4EntryCount;
-pd_entry_t *KPTphys;
-pd_entry_t *IdlePTD;
-pdpt_entry_t *IdlePDPT;
-pml4_entry_t *IdlePML4;
+int allow_64bit_proc_LDT_ops;
char *physfree;
void idt64_remap(void);
fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
{
int i;
- for (i=0; i<count; i++) {
+ for (i = 0; i < count; i++) {
base[index] = src | prot | INTEL_PTE_VALID;
src += PAGE_SIZE;
index++;
// NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
// the PCI hole (which is less 4GB but not more).
-/* Compile-time guard: NPHYSMAP is capped to 256GiB, accounting for
- * randomisation
- */
-extern int maxphymapsupported[NPHYSMAP <= (PTE_PER_PAGE/2) ? 1 : -1];
+static int
+physmap_init_L2(uint64_t *physStart, pt_entry_t **l2ptep)
+{
+ unsigned i;
+ pt_entry_t *physmapL2 = ALLOCPAGES(1);
+
+ if (physmapL2 == NULL) {
+ DBG("physmap_init_L2 page alloc failed when initting L2 for physAddr 0x%llx.\n", *physStart);
+ *l2ptep = NULL;
+ return -1;
+ }
+
+ for (i = 0; i < NPDPG; i++) {
+ physmapL2[i] = *physStart
+ | INTEL_PTE_PS
+ | INTEL_PTE_VALID
+ | INTEL_PTE_NX
+ | INTEL_PTE_WRITE;
+
+ *physStart += NBPD;
+ }
+ *l2ptep = physmapL2;
+ return 0;
+}
+
+static int
+physmap_init_L3(int startIndex, uint64_t highest_phys, uint64_t *physStart, pt_entry_t **l3ptep)
+{
+ unsigned i;
+ int ret;
+ pt_entry_t *l2pte;
+ pt_entry_t *physmapL3 = ALLOCPAGES(1); /* ALLOCPAGES bzeroes the memory */
+
+ if (physmapL3 == NULL) {
+ DBG("physmap_init_L3 page alloc failed when initting L3 for physAddr 0x%llx.\n", *physStart);
+ *l3ptep = NULL;
+ return -1;
+ }
+
+ for (i = startIndex; i < NPDPTPG && *physStart < highest_phys; i++) {
+ if ((ret = physmap_init_L2(physStart, &l2pte)) < 0) {
+ return ret;
+ }
+
+ physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(l2pte))
+ | INTEL_PTE_VALID
+ | INTEL_PTE_NX
+ | INTEL_PTE_WRITE;
+ }
+
+ *l3ptep = physmapL3;
+
+ return 0;
+}
static void
-physmap_init(void)
+physmap_init(uint8_t phys_random_L3)
{
- pt_entry_t *physmapL3 = ALLOCPAGES(1);
- struct {
- pt_entry_t entries[PTE_PER_PAGE];
- } * physmapL2 = ALLOCPAGES(NPHYSMAP);
+ pt_entry_t *l3pte;
+ int pml4_index, i;
+ int L3_start_index;
+ uint64_t physAddr = 0;
+ uint64_t highest_physaddr;
+ unsigned pdpte_count;
+
+#if DEVELOPMENT || DEBUG
+ if (kernelBootArgs->PhysicalMemorySize > K64_MAXMEM) {
+ panic("Installed physical memory exceeds configured maximum.");
+ }
+#endif
+
+ /*
+ * Add 4GB to the loader-provided physical memory size to account for MMIO space
+ * XXX in a perfect world, we'd scan PCI buses and count the max memory requested in BARs by
+ * XXX all enumerated device, then add more for hot-pluggable devices.
+ */
+ highest_physaddr = kernelBootArgs->PhysicalMemorySize + 4 * GB;
+
+ /*
+ * Calculate the number of PML4 entries we'll need. The total number of entries is
+ * pdpte_count = (((highest_physaddr) >> PDPT_SHIFT) + entropy_value +
+ * ((highest_physaddr & PDPT_MASK) == 0 ? 0 : 1))
+ * pml4e_count = pdpte_count >> (PML4_SHIFT - PDPT_SHIFT)
+ */
+ assert(highest_physaddr < (UINT64_MAX - PDPTMASK));
+ pdpte_count = (unsigned) (((highest_physaddr + PDPTMASK) >> PDPTSHIFT) + phys_random_L3);
+ kernPhysPML4EntryCount = (pdpte_count + ((1U << (PML4SHIFT - PDPTSHIFT)) - 1)) >> (PML4SHIFT - PDPTSHIFT);
+ if (kernPhysPML4EntryCount == 0) {
+ kernPhysPML4EntryCount = 1;
+ }
+ if (kernPhysPML4EntryCount > KERNEL_PHYSMAP_PML4_COUNT_MAX) {
+#if DEVELOPMENT || DEBUG
+ panic("physmap too large");
+#else
+ kprintf("[pmap] Limiting physmap to %d PML4s (was %d)\n", KERNEL_PHYSMAP_PML4_COUNT_MAX,
+ kernPhysPML4EntryCount);
+ kernPhysPML4EntryCount = KERNEL_PHYSMAP_PML4_COUNT_MAX;
+#endif
+ }
+
+ kernPhysPML4Index = KERNEL_KEXTS_INDEX - kernPhysPML4EntryCount; /* utb: KERNEL_PHYSMAP_PML4_INDEX */
+
+ /*
+ * XXX: Make sure that the addresses returned for physmapL3 and physmapL2 plus their extents
+ * are in the system-available memory range
+ */
- uint64_t i;
- uint8_t phys_random_L3 = early_random() & 0xFF;
/* We assume NX support. Mark all levels of the PHYSMAP NX
* to avoid granting executability via a single bit flip.
}
#endif /* DEVELOPMENT || DEBUG */
- for(i = 0; i < NPHYSMAP; i++) {
- physmapL3[i + phys_random_L3] =
- ((uintptr_t)ID_MAP_VTOP(&physmapL2[i]))
- | INTEL_PTE_VALID
- | INTEL_PTE_NX
- | INTEL_PTE_WRITE;
-
- uint64_t j;
- for(j = 0; j < PTE_PER_PAGE; j++) {
- physmapL2[i].entries[j] =
- ((i * PTE_PER_PAGE + j) << PDSHIFT)
- | INTEL_PTE_PS
- | INTEL_PTE_VALID
- | INTEL_PTE_NX
- | INTEL_PTE_WRITE;
+ L3_start_index = phys_random_L3;
+
+ for (pml4_index = kernPhysPML4Index;
+ pml4_index < (kernPhysPML4Index + kernPhysPML4EntryCount) && physAddr < highest_physaddr;
+ pml4_index++) {
+ if (physmap_init_L3(L3_start_index, highest_physaddr, &physAddr, &l3pte) < 0) {
+ panic("Physmap page table initialization failed");
+ /* NOTREACHED */
}
+
+ L3_start_index = 0;
+
+ IdlePML4[pml4_index] = ((uintptr_t)ID_MAP_VTOP(l3pte))
+ | INTEL_PTE_VALID
+ | INTEL_PTE_NX
+ | INTEL_PTE_WRITE;
}
- IdlePML4[KERNEL_PHYSMAP_PML4_INDEX] =
- ((uintptr_t)ID_MAP_VTOP(physmapL3))
- | INTEL_PTE_VALID
- | INTEL_PTE_NX
- | INTEL_PTE_WRITE;
+ physmap_base = KVADDR(kernPhysPML4Index, phys_random_L3, 0, 0);
+ /*
+ * physAddr contains the last-mapped physical address, so that's what we
+ * add to physmap_base to derive the ending VA for the physmap.
+ */
+ physmap_max = physmap_base + physAddr;
- physmap_base = KVADDR(KERNEL_PHYSMAP_PML4_INDEX, phys_random_L3, 0, 0);
- physmap_max = physmap_base + NPHYSMAP * GB;
DBG("Physical address map base: 0x%qx\n", physmap_base);
- DBG("Physical map idlepml4[%d]: 0x%llx\n",
- KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
+ for (i = kernPhysPML4Index; i < (kernPhysPML4Index + kernPhysPML4EntryCount); i++) {
+ DBG("Physical map idlepml4[%d]: 0x%llx\n", i, IdlePML4[i]);
+ }
}
-void doublemap_init(void);
+void doublemap_init(uint8_t);
static void
Idle_PTs_init(void)
{
+ uint64_t rand64;
+
/* Allocate the "idle" kernel page tables: */
- KPTphys = ALLOCPAGES(NKPT); /* level 1 */
- IdlePTD = ALLOCPAGES(NPGPTD); /* level 2 */
- IdlePDPT = ALLOCPAGES(1); /* level 3 */
- IdlePML4 = ALLOCPAGES(1); /* level 4 */
+ KPTphys = ALLOCPAGES(NKPT); /* level 1 */
+ IdlePTD = ALLOCPAGES(NPGPTD); /* level 2 */
+ IdlePDPT = ALLOCPAGES(1); /* level 3 */
+ IdlePML4 = ALLOCPAGES(1); /* level 4 */
// Fill the lowest level with everything up to physfree
fillkpt(KPTphys,
- INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
+ INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
/* IdlePTD */
fillkpt(IdlePTD,
- INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
+ INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
// IdlePDPT entries
fillkpt(IdlePDPT,
- INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
+ INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
// IdlePML4 single entry for kernel space.
fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
- INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
-
+ INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
+
postcode(VSTART_PHYSMAP_INIT);
- physmap_init();
- doublemap_init();
+ /*
+ * early_random() cannot be called more than one time before the cpu's
+ * gsbase is initialized, so use the full 64-bit value to extract the
+ * two 8-bit entropy values needed for address randomization.
+ */
+ rand64 = early_random();
+ physmap_init(rand64 & 0xFF);
+ doublemap_init((rand64 >> 8) & 0xFF);
idt64_remap();
postcode(VSTART_SET_CR3);
// Switch to the page tables..
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
-
}
extern void vstart_trap_handler;
-#define BOOT_TRAP_VECTOR(t) \
- [t] = { \
- (uintptr_t) &vstart_trap_handler, \
- KERNEL64_CS, \
- 0, \
- ACC_P|ACC_PL_K|ACC_INTR_GATE, \
- 0 \
+#define BOOT_TRAP_VECTOR(t) \
+ [t] = { \
+ (uintptr_t) &vstart_trap_handler, \
+ KERNEL64_CS, \
+ 0, \
+ ACC_P|ACC_PL_K|ACC_INTR_GATE, \
+ 0 \
},
/* Recursive macro to iterate 0..31 */
-#define L0(x,n) x(n)
-#define L1(x,n) L0(x,n-1) L0(x,n)
-#define L2(x,n) L1(x,n-2) L1(x,n)
-#define L3(x,n) L2(x,n-4) L2(x,n)
-#define L4(x,n) L3(x,n-8) L3(x,n)
-#define L5(x,n) L4(x,n-16) L4(x,n)
+#define L0(x, n) x(n)
+#define L1(x, n) L0(x,n-1) L0(x,n)
+#define L2(x, n) L1(x,n-2) L1(x,n)
+#define L3(x, n) L2(x,n-4) L2(x,n)
+#define L4(x, n) L3(x,n-8) L3(x,n)
+#define L5(x, n) L4(x,n-16) L4(x,n)
#define FOR_0_TO_31(x) L5(x,31)
/*
* All traps point to a common handler.
*/
struct fake_descriptor64 master_boot_idt64[IDTSZ]
- __attribute__((section("__HIB,__desc")))
- __attribute__((aligned(PAGE_SIZE))) = {
+__attribute__((section("__HIB,__desc")))
+__attribute__((aligned(PAGE_SIZE))) = {
FOR_0_TO_31(BOOT_TRAP_VECTOR)
};
static void
vstart_idt_init(void)
{
- x86_64_desc_register_t vstart_idt = {
- sizeof(master_boot_idt64),
- master_boot_idt64 };
-
+ x86_64_desc_register_t vstart_idt = {
+ sizeof(master_boot_idt64),
+ master_boot_idt64
+ };
+
fix_desc64(master_boot_idt64, 32);
lidt((void *)&vstart_idt);
}
/*
* vstart() is called in the natural mode (64bit for K64, 32 for K32)
- * on a set of bootstrap pagetables which use large, 2MB pages to map
+ * on a set of bootstrap pagetables which use large, 2MB pages to map
* all of physical memory in both. See idle_pt.c for details.
*
- * In K64 this identity mapping is mirrored the top and bottom 512GB
+ * In K64 this identity mapping is mirrored the top and bottom 512GB
* slots of PML4.
*
* The bootstrap processor called with argument boot_args_start pointing to
void
vstart(vm_offset_t boot_args_start)
{
- boolean_t is_boot_cpu = !(boot_args_start == 0);
- int cpu = 0;
- uint32_t lphysfree;
+ boolean_t is_boot_cpu = !(boot_args_start == 0);
+ int cpu = 0;
+ uint32_t lphysfree;
postcode(VSTART_ENTRY);
vstart_idt_init();
postcode(VSTART_IDT_INIT);
+ /*
+ * Ensure that any %gs-relative access results in an immediate fault
+ * until gsbase is properly initialized below
+ */
+ wrmsr64(MSR_IA32_GS_BASE, EARLY_GSBASE_MAGIC);
+
/*
* Get startup parameters.
*/
kernelBootArgs = (boot_args *)boot_args_start;
lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
- physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1));
+ physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
#if DEVELOPMENT || DEBUG
pal_serial_init();
DBG("ksize 0x%x\n", kernelBootArgs->ksize);
DBG("physfree %p\n", physfree);
DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
- kernelBootArgs,
- &kernelBootArgs->ksize,
- &kernelBootArgs->kaddr);
+ kernelBootArgs,
+ &kernelBootArgs->ksize,
+ &kernelBootArgs->kaddr);
DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize);
/*
postcode(VSTART_CPU_MODE_INIT);
cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
- * invoked on the APs
- * via i386_init_slave()
- */
+ * invoked on the APs
+ * via i386_init_slave()
+ */
} else {
/* Switch to kernel's page tables (from the Boot PTs) */
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
/* Find our logical cpu number */
- cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
+ cpu = lapic_to_cpu[(LAPIC_READ(ID) >> LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
cpu_desc_load(cpu_datap(cpu));
}
+ early_boot = 0;
postcode(VSTART_EXIT);
x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
- : (uintptr_t) i386_init_slave,
- cpu_datap(cpu)->cpu_int_stack_top);
+ : (uintptr_t) i386_init_slave,
+ cpu_datap(cpu)->cpu_int_stack_top);
}
void
void
i386_init(void)
{
- unsigned int maxmem;
- uint64_t maxmemtouse;
- unsigned int cpus = 0;
- boolean_t fidn;
- boolean_t IA32e = TRUE;
+ unsigned int maxmem;
+ uint64_t maxmemtouse;
+ unsigned int cpus = 0;
+ boolean_t fidn;
+ boolean_t IA32e = TRUE;
postcode(I386_INIT_ENTRY);
pal_i386_init();
tsc_init();
- rtclock_early_init(); /* mach_absolute_time() now functionsl */
+ rtclock_early_init(); /* mach_absolute_time() now functionsl */
kernel_debug_string_early("i386_init");
pstate_trace();
postcode(CPU_INIT_D);
- printf_init(); /* Init this in case we need debugger */
- panic_init(); /* Init this in case we need debugger */
+ printf_init(); /* Init this in case we need debugger */
+ panic_init(); /* Init this in case we need debugger */
/* setup debugging output if one has been chosen */
kernel_debug_string_early("PE_init_kprintf");
kernel_debug_string_early("kernel_early_bootstrap");
kernel_early_bootstrap();
- if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
+ if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof(dgWork.dgFlags))) {
dgWork.dgFlags = 0;
+ }
+
+ if (!PE_parse_boot_argn("ldt64", &allow_64bit_proc_LDT_ops,
+ sizeof(allow_64bit_proc_LDT_ops))) {
+ allow_64bit_proc_LDT_ops = 0;
+ }
serialmode = 0;
if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
if (force_sync) {
serialmode |= SERIALMODE_SYNCDRAIN;
kprintf(
- "WARNING: Forcing uart driver to output synchronously."
- "printf()s/IOLogs will impact kernel performance.\n"
- "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
+ "WARNING: Forcing uart driver to output synchronously."
+ "printf()s/IOLogs will impact kernel performance.\n"
+ "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
}
}
}
kprintf("version_variant = %s\n", version_variant);
kprintf("version = %s\n", version);
-
- if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem)))
+
+ if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
maxmemtouse = 0;
- else
- maxmemtouse = ((uint64_t)maxmem) * MB;
+ } else {
+ maxmemtouse = ((uint64_t)maxmem) * MB;
+ }
- if (PE_parse_boot_argn("cpus", &cpus, sizeof (cpus))) {
- if ((0 < cpus) && (cpus < max_ncpus))
- max_ncpus = cpus;
+ if (PE_parse_boot_argn("cpus", &cpus, sizeof(cpus))) {
+ if ((0 < cpus) && (cpus < max_ncpus)) {
+ max_ncpus = cpus;
+ }
}
/*
* debug support for > 4G systems
*/
- PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode));
- if (vm_himemory_mode != 0)
- kprintf("himemory_mode: %d\n", vm_himemory_mode);
+ PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof(vm_himemory_mode));
+ if (!vm_himemory_mode) {
+ kprintf("himemory_mode disabled\n");
+ }
- if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof (fidn)))
+ if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof(fidn))) {
force_immediate_debugger_NMI = FALSE;
- else
+ } else {
force_immediate_debugger_NMI = fidn;
+ }
#if DEBUG
nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);
&urgency_notification_assert_abstime_threshold,
sizeof(urgency_notification_assert_abstime_threshold));
- if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
+ if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD)) {
nx_enabled = 0;
+ }
- /*
+ /*
* VM initialization, after this we're using page tables...
* Thn maximum number of cpus must be set beforehand.
*/
static void
do_init_slave(boolean_t fast_restart)
{
- void *init_param = FULL_SLAVE_INIT;
+ void *init_param = FULL_SLAVE_INIT;
postcode(I386_INIT_SLAVE);
if (!fast_restart) {
/* Ensure that caching and write-through are enabled */
- set_cr0(get_cr0() & ~(CR0_NW|CR0_CD));
-
+ set_cr0(get_cr0() & ~(CR0_NW | CR0_CD));
+
DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
get_cpu_number(), get_cpu_phys_number());
-
+
assert(!ml_get_interrupts_enabled());
-
+
cpu_syscall_init(current_cpu_datap());
pmap_cpu_init();
-
+
#if CONFIG_MCA
mca_cpu_init();
#endif
-
+
LAPIC_INIT();
lapic_configure();
LAPIC_DUMP();
LAPIC_CPU_MAP_DUMP();
-
+
init_fpu();
-
+
#if CONFIG_MTRR
mtrr_update_cpu();
#endif
/* update CPU microcode */
ucode_update_wake();
- } else
- init_param = FAST_SLAVE_INIT;
+
+ /* Do CPU workarounds after the microcode update */
+ cpuid_do_was();
+ } else {
+ init_param = FAST_SLAVE_INIT;
+ }
#if CONFIG_VMX
/* resume VT operation */
#endif
#if CONFIG_MTRR
- if (!fast_restart)
- pat_init();
+ if (!fast_restart) {
+ pat_init();
+ }
#endif
- cpu_thread_init(); /* not strictly necessary */
+ cpu_thread_init(); /* not strictly necessary */
+
+ cpu_init(); /* Sets cpu_running which starter cpu waits for */
+ slave_main(init_param);
- cpu_init(); /* Sets cpu_running which starter cpu waits for */
- slave_main(init_param);
-
- panic("do_init_slave() returned from slave_main()");
+ panic("do_init_slave() returned from slave_main()");
}
/*
void
i386_init_slave(void)
{
- do_init_slave(FALSE);
+ do_init_slave(FALSE);
}
/*
void
i386_init_slave_fast(void)
{
- do_init_slave(TRUE);
+ do_init_slave(TRUE);
}
#include <libkern/kernel_mach_header.h>
extern uint64_t idt64_hndl_table0[];
-void doublemap_init(void) {
+void
+doublemap_init(uint8_t randL3)
+{
dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries
dblallocs++;
} * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries
dblallocs++;
- dblmapL3[0] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
+ dblmapL3[randL3] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
| INTEL_PTE_VALID
| INTEL_PTE_WRITE;
assert((hdescb & 0xFFF) == 0);
/* Mirror HIB translations into the double-mapped pagetable subtree*/
- for(int i = 0; hdescc < hdesce; i++) {
+ for (int i = 0; hdescc < hdesce; i++) {
struct {
pt_entry_t entries[PTE_PER_PAGE];
} * dblmapL1 = ALLOCPAGES(1);
if ((hdescc >= thdescb) && (hdescc < thdesce)) {
/* executable */
} else {
- template |= INTEL_PTE_WRITE | INTEL_PTE_NX ; /* Writeable, NX */
+ template |= INTEL_PTE_WRITE | INTEL_PTE_NX; /* Writeable, NX */
}
dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
hdescc += PAGE_SIZE;
IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
- dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, dblmapL3, 0, 0);
+ dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, randL3, 0, 0);
dblmap_max = dblmap_base + hdescszr;
/* Calculate the double-map distance, which accounts for the current
* KASLR slide
* programmed into GSBASE, to the "shadows" in the doublemapped
* region. These are not aliases, but separate physical allocations
* containing data required in the doublemapped trampolines.
-*/
+ */
idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
DBG("Double map base: 0x%qx\n", dblmap_base);
/* Use of this routine is expected to be synchronized by callers
* Creates non-executable aliases.
*/
-vm_offset_t dyn_dblmap(vm_offset_t cva, vm_offset_t sz) {
+vm_offset_t
+dyn_dblmap(vm_offset_t cva, vm_offset_t sz)
+{
vm_offset_t ava = dblmap_max;
assert((sz & PAGE_MASK) == 0);
pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
dblmap_max += sz;
- return (ava - cva);
+ return ava - cva;
}
/* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect
* control to the double-mapped interrupt vectors. The IDTR proper will be
* programmed via cpu_desc_load()
*/
-void idt64_remap(void) {
+void
+idt64_remap(void)
+{
for (int i = 0; i < IDTSZ; i++) {
master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
}
projects / apple / xnu.git / blobdiff