X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..b226f5e54a60dc81db17b1260381d7dbfea3cdf1:/osfmk/i386/i386_init.c diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c index 445c6afed..8eb6b7edf 100644 --- a/osfmk/i386/i386_init.c +++ b/osfmk/i386/i386_init.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2009 Apple Inc. All rights reserved. + * Copyright (c) 2003-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -54,8 +54,6 @@ * the rights to redistribute these changes. */ -#include -#include #include @@ -74,44 +72,48 @@ #include #include #include +#include #include #include #include #include +#include #include #include -#include #include #include #include #include #include #include +#if CONFIG_MTRR #include +#endif #include #if CONFIG_MCA #include #endif +#include #include #include #include #include #include /* LcksOpts */ -#ifdef __i386__ -#include -#if MACH_KDB -#include -#endif +#if DEBUG +#include #endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +#include + #if DEBUG #define DBG(x...) kprintf(x) #else #define DBG(x...) #endif -#if MACH_KDB -#include -#endif /* MACH_KDB */ int debug_task; @@ -122,22 +124,20 @@ extern const char version[]; extern const char version_variant[]; extern int nx_enabled; -extern int noVMX; /* if set, rosetta should not emulate altivec */ - -#ifdef __x86_64__ -extern void *low_eintstack; -#endif - -extern void serial_init(void); +/* + * Set initial values so that ml_phys_* routines can use the booter's ID mapping + * to touch physical space before the kernel's physical aperture exists. + */ +uint64_t physmap_base = 0; +uint64_t physmap_max = 4*GB; -void *KPTphys; +pd_entry_t *KPTphys; pd_entry_t *IdlePTD; -#ifdef __i386__ -pd_entry_t *IdlePDPT64; -#endif - +pdpt_entry_t *IdlePDPT; +pml4_entry_t *IdlePML4; char *physfree; +void idt64_remap(void); /* * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init() @@ -149,9 +149,7 @@ ALLOCPAGES(int npages) uintptr_t tmp = (uintptr_t)physfree; bzero(physfree, npages * PAGE_SIZE); physfree += npages * PAGE_SIZE; -#ifdef __x86_64__ tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK; -#endif return (void *)tmp; } @@ -166,17 +164,16 @@ fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count) } } -extern vm_offset_t first_avail; +extern pmap_paddr_t first_avail; -#ifdef __x86_64__ int break_kprintf = 0; uint64_t x86_64_pre_sleep(void) { IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX]; - uint64_t oldcr3 = get_cr3(); - set_cr3((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4)); + uint64_t oldcr3 = get_cr3_raw(); + set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4)); return oldcr3; } @@ -184,24 +181,21 @@ void x86_64_post_sleep(uint64_t new_cr3) { IdlePML4[0] = 0; - set_cr3((uint32_t) new_cr3); + set_cr3_raw((uint32_t) new_cr3); } -#endif - -#ifdef __i386__ -#define ID_MAP_VTOP(x) x -#endif -#ifdef __x86_64__ // Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address // NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account // the PCI hole (which is less 4GB but not more). -#define NPHYSMAP MAX(K64_MAXMEM/GB + 4, 4) -// Compile-time guard: -extern int maxphymapsupported[NPHYSMAP <= PTE_PER_PAGE ? 1 : -1]; + +/* Compile-time guard: NPHYSMAP is capped to 256GiB, accounting for + * randomisation + */ +extern int maxphymapsupported[NPHYSMAP <= (PTE_PER_PAGE/2) ? 1 : -1]; + static void physmap_init(void) { @@ -210,27 +204,53 @@ physmap_init(void) pt_entry_t entries[PTE_PER_PAGE]; } * physmapL2 = ALLOCPAGES(NPHYSMAP); - uintptr_t i; - for(i=0;i= 0x80000001) { + do_cpuid(0x80000001, reg); + assert(reg[edx] & CPUID_EXTFEATURE_XD); + } +#endif /* DEVELOPMENT || DEBUG */ + + for(i = 0; i < NPHYSMAP; i++) { + physmapL3[i + phys_random_L3] = + ((uintptr_t)ID_MAP_VTOP(&physmapL2[i])) | INTEL_PTE_VALID + | INTEL_PTE_NX | INTEL_PTE_WRITE; - uintptr_t j; - for(j=0;j> PDESHIFT,1); -#endif // Fill the lowest level with everything up to physfree fillkpt(KPTphys, - INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT)); + INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT)); - // Rewrite the 2nd-lowest level to point to pages of KPTphys. - // This was previously filled statically by idle_pt.c, and thus - // must be done after the KPTphys fill since IdlePTD is in use + /* IdlePTD */ fillkpt(IdlePTD, - INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT); + INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT); // IdlePDPT entries -#ifdef __i386__ - fillkpt(IdlePDPT, 0, (uintptr_t)IdlePTD, 0, NPGPTD); -#else - fillkpt(IdlePDPT, INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD); -#endif + fillkpt(IdlePDPT, + INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD); + + // IdlePML4 single entry for kernel space. + fillkpt(IdlePML4 + KERNEL_PML4_INDEX, + INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1); + + postcode(VSTART_PHYSMAP_INIT); + + physmap_init(); + doublemap_init(); + idt64_remap(); + + postcode(VSTART_SET_CR3); + + // Switch to the page tables.. + set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4)); + +} + +extern void vstart_trap_handler; + +#define BOOT_TRAP_VECTOR(t) \ + [t] = { \ + (uintptr_t) &vstart_trap_handler, \ + KERNEL64_CS, \ + 0, \ + ACC_P|ACC_PL_K|ACC_INTR_GATE, \ + 0 \ + }, + +/* Recursive macro to iterate 0..31 */ +#define L0(x,n) x(n) +#define L1(x,n) L0(x,n-1) L0(x,n) +#define L2(x,n) L1(x,n-2) L1(x,n) +#define L3(x,n) L2(x,n-4) L2(x,n) +#define L4(x,n) L3(x,n-8) L3(x,n) +#define L5(x,n) L4(x,n-16) L4(x,n) +#define FOR_0_TO_31(x) L5(x,31) + +/* + * Bootstrap IDT. Active only during early startup. + * Only the trap vectors are defined since interrupts are masked. + * All traps point to a common handler. + */ +struct fake_descriptor64 master_boot_idt64[IDTSZ] + __attribute__((section("__HIB,__desc"))) + __attribute__((aligned(PAGE_SIZE))) = { + FOR_0_TO_31(BOOT_TRAP_VECTOR) +}; - // Flush the TLB now we're done rewriting the page tables.. - set_cr3(get_cr3()); +static void +vstart_idt_init(void) +{ + x86_64_desc_register_t vstart_idt = { + sizeof(master_boot_idt64), + master_boot_idt64 }; + + fix_desc64(master_boot_idt64, 32); + lidt((void *)&vstart_idt); } /* @@ -285,24 +347,32 @@ Idle_PTs_init(void) * Non-bootstrap processors are called with argument boot_args_start NULL. * These processors switch immediately to the existing kernel page tables. */ +__attribute__((noreturn)) void vstart(vm_offset_t boot_args_start) { boolean_t is_boot_cpu = !(boot_args_start == 0); - int cpu; - uint32_t lphysfree; + int cpu = 0; + uint32_t lphysfree; postcode(VSTART_ENTRY); if (is_boot_cpu) { + /* + * Set-up temporary trap handlers during page-table set-up. + */ + vstart_idt_init(); + postcode(VSTART_IDT_INIT); + /* * Get startup parameters. */ kernelBootArgs = (boot_args *)boot_args_start; lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize; physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1)); -#if DEBUG - serial_init(); + +#if DEVELOPMENT || DEBUG + pal_serial_init(); #endif DBG("revision 0x%x\n", kernelBootArgs->Revision); DBG("version 0x%x\n", kernelBootArgs->Version); @@ -316,71 +386,70 @@ vstart(vm_offset_t boot_args_start) kernelBootArgs, &kernelBootArgs->ksize, &kernelBootArgs->kaddr); + DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize); - postcode(PSTART_PAGE_TABLES); + /* + * Setup boot args given the physical start address. + * Note: PE_init_platform needs to be called before Idle_PTs_init + * because access to the DeviceTree is required to read the + * random seed before generating a random physical map slide. + */ + kernelBootArgs = (boot_args *) + ml_static_ptovirt(boot_args_start); + DBG("i386_init(0x%lx) kernelBootArgs=%p\n", + (unsigned long)boot_args_start, kernelBootArgs); + +#if KASAN + kasan_reserve_memory(kernelBootArgs); +#endif + + PE_init_platform(FALSE, kernelBootArgs); + postcode(PE_INIT_PLATFORM_D); Idle_PTs_init(); + postcode(VSTART_IDLE_PTS_INIT); + +#if KASAN + /* Init kasan and map whatever was stolen from physfree */ + kasan_init(); + kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree)); +#endif + +#if MONOTONIC + mt_early_init(); +#endif /* MONOTONIC */ first_avail = (vm_offset_t)ID_MAP_VTOP(physfree); - cpu = 0; + cpu_data_alloc(TRUE); + + cpu_desc_init(cpu_datap(0)); + postcode(VSTART_CPU_DESC_INIT); + cpu_desc_load(cpu_datap(0)); + + postcode(VSTART_CPU_MODE_INIT); + cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be + * invoked on the APs + * via i386_init_slave() + */ } else { + /* Switch to kernel's page tables (from the Boot PTs) */ + set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4)); /* Find our logical cpu number */ cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK]; + DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE)); + cpu_desc_load(cpu_datap(cpu)); } - if(is_boot_cpu) cpu_data_alloc(TRUE); -#ifdef __x86_64__ - if(is_boot_cpu) - cpu_desc_init64(cpu_datap(cpu)); - cpu_desc_load64(cpu_datap(cpu)); -#else - if(is_boot_cpu) - cpu_desc_init(cpu_datap(cpu)); - cpu_desc_load(cpu_datap(cpu)); -#endif - cpu_mode_init(current_cpu_datap()); - - /* enable NX/XD */ - if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) - wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_NXE); - DBG("vstart() NX/XD enabled\n"); - - -#ifdef __x86_64__ - /* Done with identity mapping */ - IdlePML4[0] = 0; -#endif - postcode(VSTART_EXIT); -#ifdef __i386__ - if (is_boot_cpu) - i386_init(boot_args_start); - else - i386_init_slave(); - /*NOTREACHED*/ -#else - /* We need to switch to a new per-cpu stack, but we must do this atomically with - * the call to ensure the compiler doesn't assume anything about the stack before - * e.g. tail-call optimisations - */ - if (is_boot_cpu) - { - asm volatile( - "mov %1, %%rdi;" - "mov %0, %%rsp;" - "call _i386_init;" : : "r" - (cpu_datap(cpu)->cpu_int_stack_top), "r" (boot_args_start)); - } - else - { - asm volatile( - "mov %0, %%rsp;" - "call _i386_init_slave;" : : "r" - (cpu_datap(cpu)->cpu_int_stack_top)); - } - /*NOTREACHED*/ -#endif + x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init + : (uintptr_t) i386_init_slave, + cpu_datap(cpu)->cpu_int_stack_top); +} + +void +pstate_trace(void) +{ } /* @@ -388,63 +457,68 @@ vstart(vm_offset_t boot_args_start) * set up. */ void -i386_init(vm_offset_t boot_args_start) +i386_init(void) { unsigned int maxmem; uint64_t maxmemtouse; unsigned int cpus = 0; boolean_t fidn; -#ifdef __i386__ - boolean_t legacy_mode; -#endif boolean_t IA32e = TRUE; postcode(I386_INIT_ENTRY); + pal_i386_init(); + tsc_init(); + rtclock_early_init(); /* mach_absolute_time() now functionsl */ + + kernel_debug_string_early("i386_init"); + pstate_trace(); + #if CONFIG_MCA /* Initialize machine-check handling */ mca_cpu_init(); #endif - /* - * Setup boot args given the physical start address. - */ - kernelBootArgs = (boot_args *) - ml_static_ptovirt(boot_args_start); - DBG("i386_init(0x%lx) kernelBootArgs=%p\n", - (unsigned long)boot_args_start, kernelBootArgs); - master_cpu = 0; cpu_init(); postcode(CPU_INIT_D); - - PE_init_platform(FALSE, kernelBootArgs); - postcode(PE_INIT_PLATFORM_D); - - printf_init(); /* Init this in case we need debugger */ panic_init(); /* Init this in case we need debugger */ - /* setup debugging output if one has been chosen */ + kernel_debug_string_early("PE_init_kprintf"); PE_init_kprintf(FALSE); + kernel_debug_string_early("kernel_early_bootstrap"); + kernel_early_bootstrap(); + if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags))) dgWork.dgFlags = 0; serialmode = 0; - if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) { + if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* We want a serial keyboard and/or console */ kprintf("Serial mode specified: %08X\n", serialmode); + int force_sync = serialmode & SERIALMODE_SYNCDRAIN; + if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) { + if (force_sync) { + serialmode |= SERIALMODE_SYNCDRAIN; + kprintf( + "WARNING: Forcing uart driver to output synchronously." + "printf()s/IOLogs will impact kernel performance.\n" + "You are advised to avoid using 'drain_uart_sync' boot-arg.\n"); + } + } } - if(serialmode & 1) { + if (serialmode & SERIALMODE_OUTPUT) { (void)switch_to_serial_console(); - disableConsoleOutput = FALSE; /* Allow printfs to happen */ + disableConsoleOutput = FALSE; /* Allow printfs to happen */ } /* setup console output */ + kernel_debug_string_early("PE_init_printf"); PE_init_printf(FALSE); kprintf("version_variant = %s\n", version_variant); @@ -460,62 +534,49 @@ i386_init(vm_offset_t boot_args_start) max_ncpus = cpus; } - /* * debug support for > 4G systems */ - if (!PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode))) - vm_himemory_mode = 0; + PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode)); + if (vm_himemory_mode != 0) + kprintf("himemory_mode: %d\n", vm_himemory_mode); if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof (fidn))) force_immediate_debugger_NMI = FALSE; else force_immediate_debugger_NMI = fidn; -#ifdef __i386__ - /* - * At this point we check whether we are a 64-bit processor - * and that we're not restricted to legacy mode, 32-bit operation. - */ - if (cpuid_extfeatures() & CPUID_EXTFEATURE_EM64T) { - kprintf("EM64T supported"); - if (PE_parse_boot_argn("-legacy", &legacy_mode, sizeof (legacy_mode))) { - kprintf(" but legacy mode forced\n"); - IA32e = FALSE; - } else { - kprintf(" and will be enabled\n"); - } - } else - IA32e = FALSE; + +#if DEBUG + nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold); #endif + PE_parse_boot_argn("urgency_notification_abstime", + &urgency_notification_assert_abstime_threshold, + sizeof(urgency_notification_assert_abstime_threshold)); if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD)) nx_enabled = 0; - /* Obtain "lcks" options:this currently controls lock statistics */ - if (!PE_parse_boot_argn("lcks", &LcksOpts, sizeof (LcksOpts))) - LcksOpts = 0; - /* * VM initialization, after this we're using page tables... - * The maximum number of cpus must be set beforehand. + * Thn maximum number of cpus must be set beforehand. */ + kernel_debug_string_early("i386_vm_init"); i386_vm_init(maxmemtouse, IA32e, kernelBootArgs); - if ( ! PE_parse_boot_argn("novmx", &noVMX, sizeof (noVMX))) - noVMX = 0; /* OK to support Altivec in rosetta? */ - - tsc_init(); - power_management_init(); - - PE_init_platform(TRUE, kernelBootArgs); - /* create the console for verbose or pretty mode */ + /* Note: doing this prior to tsc_init() allows for graceful panic! */ + PE_init_platform(TRUE, kernelBootArgs); PE_create_console(); + kernel_debug_string_early("power_management_init"); + power_management_init(); processor_bootstrap(); thread_bootstrap(); + pstate_trace(); + kernel_debug_string_early("machine_startup"); machine_startup(); + pstate_trace(); } static void @@ -534,42 +595,41 @@ do_init_slave(boolean_t fast_restart) assert(!ml_get_interrupts_enabled()); - cpu_mode_init(current_cpu_datap()); + cpu_syscall_init(current_cpu_datap()); + pmap_cpu_init(); #if CONFIG_MCA mca_cpu_init(); #endif + LAPIC_INIT(); lapic_configure(); LAPIC_DUMP(); LAPIC_CPU_MAP_DUMP(); init_fpu(); +#if CONFIG_MTRR mtrr_update_cpu(); +#endif + /* update CPU microcode */ + ucode_update_wake(); } else init_param = FAST_SLAVE_INIT; #if CONFIG_VMX /* resume VT operation */ - vmx_resume(); + vmx_resume(FALSE); #endif +#if CONFIG_MTRR if (!fast_restart) pat_init(); - - cpu_thread_init(); /* not strictly necessary */ - -#ifdef __x86_64__ - /* Re-zero the identity-map for the idle PT's. This MUST be done before - * cpu_running is set so that other slaves can set up their own - * identity-map */ - if (!fast_restart) - IdlePML4[0] = 0; #endif - cpu_init(); /* Sets cpu_running which starter cpu waits for */ + cpu_thread_init(); /* not strictly necessary */ + cpu_init(); /* Sets cpu_running which starter cpu waits for */ slave_main(init_param); panic("do_init_slave() returned from slave_main()"); @@ -599,4 +659,116 @@ i386_init_slave_fast(void) do_init_slave(TRUE); } +#include + +/* TODO: Evaluate global PTEs for the double-mapped translations */ + +uint64_t dblmap_base, dblmap_max; +kernel_segment_command_t *hdescseg; + +pt_entry_t *dblmapL3; +unsigned int dblallocs; +uint64_t dblmap_dist; +extern uint64_t idt64_hndl_table0[]; + +void doublemap_init(void) { + dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries + dblallocs++; + + struct { + pt_entry_t entries[PTE_PER_PAGE]; + } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries + dblallocs++; + + dblmapL3[0] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0])) + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + + hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB"); + + vm_offset_t hdescb = hdescseg->vmaddr; + unsigned long hdescsz = hdescseg->vmsize; + unsigned long hdescszr = round_page_64(hdescsz); + vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr; + + kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text"); + vm_offset_t thdescb = thdescsect->addr; + unsigned long thdescsz = thdescsect->size; + unsigned long thdescszr = round_page_64(thdescsz); + vm_offset_t thdesce = thdescb + thdescszr; + + assert((hdescb & 0xFFF) == 0); + /* Mirror HIB translations into the double-mapped pagetable subtree*/ + for(int i = 0; hdescc < hdesce; i++) { + struct { + pt_entry_t entries[PTE_PER_PAGE]; + } * dblmapL1 = ALLOCPAGES(1); + dblallocs++; + dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF; + int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE); + for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) { + uint64_t template = INTEL_PTE_VALID; + if ((hdescc >= thdescb) && (hdescc < thdesce)) { + /* executable */ + } else { + template |= INTEL_PTE_WRITE | INTEL_PTE_NX ; /* Writeable, NX */ + } + dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template; + hdescc += PAGE_SIZE; + } + } + + IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF; + + dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, dblmapL3, 0, 0); + dblmap_max = dblmap_base + hdescszr; + /* Calculate the double-map distance, which accounts for the current + * KASLR slide + */ + + dblmap_dist = dblmap_base - hdescb; + idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]); + idt64_hndl_table0[6] = (uint64_t)(uintptr_t)&kernel_stack_mask; + + extern cpu_data_t cpshadows[], scdatas[]; + uintptr_t cd1 = (uintptr_t) &cpshadows[0]; + uintptr_t cd2 = (uintptr_t) &scdatas[0]; +/* Record the displacement from the kernel's per-CPU data pointer, eventually + * programmed into GSBASE, to the "shadows" in the doublemapped + * region. These are not aliases, but separate physical allocations + * containing data required in the doublemapped trampolines. +*/ + idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2; + + DBG("Double map base: 0x%qx\n", dblmap_base); + DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]); + assert(LDTSZ > LDTSZ_MIN); +} + +vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t); + +#include + +/* Use of this routine is expected to be synchronized by callers + * Creates non-executable aliases. + */ +vm_offset_t dyn_dblmap(vm_offset_t cva, vm_offset_t sz) { + vm_offset_t ava = dblmap_max; + + assert((sz & PAGE_MASK) == 0); + assert(cva != 0); + + pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP); + dblmap_max += sz; + return (ava - cva); +} +/* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect + * control to the double-mapped interrupt vectors. The IDTR proper will be + * programmed via cpu_desc_load() + */ +void idt64_remap(void) { + for (int i = 0; i < IDTSZ; i++) { + master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64); + } +}