+/*
+ * Copyright (c) 2007-2009 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+#include <debug.h>
+#include <mach_ldebug.h>
+#include <mach_kdp.h>
+
+#include <kern/misc_protos.h>
+#include <kern/thread.h>
+#include <kern/timer_queue.h>
+#include <kern/processor.h>
+#include <kern/startup.h>
+#include <kern/debug.h>
+#include <prng/random.h>
+#include <machine/machine_routines.h>
+#include <machine/commpage.h>
+/* ARM64_TODO unify boot.h */
+#if __arm64__
+#include <pexpert/arm64/boot.h>
+#elif __arm__
+#include <pexpert/arm/boot.h>
+#else
+#error Unsupported arch
+#endif
+#include <pexpert/arm/consistent_debug.h>
+#include <pexpert/device_tree.h>
+#include <arm/proc_reg.h>
+#include <arm/pmap.h>
+#include <arm/caches_internal.h>
+#include <arm/cpu_internal.h>
+#include <arm/cpu_data_internal.h>
+#include <arm/misc_protos.h>
+#include <arm/machine_cpu.h>
+#include <arm/rtclock.h>
+#include <vm/vm_map.h>
+
+#include <libkern/kernel_mach_header.h>
+#include <libkern/stack_protector.h>
+#include <libkern/section_keywords.h>
+#include <san/kasan.h>
+
+#include <pexpert/pexpert.h>
+
+#include <console/serial_protos.h>
+
+#if CONFIG_TELEMETRY
+#include <kern/telemetry.h>
+#endif
+#if MONOTONIC
+#include <kern/monotonic.h>
+#endif /* MONOTONIC */
+
+extern void patch_low_glo(void);
+extern int serial_init(void);
+extern void sleep_token_buffer_init(void);
+
+extern vm_offset_t intstack_top;
+extern vm_offset_t fiqstack_top;
+#if __arm64__
+extern vm_offset_t excepstack_top;
+#endif
+
+extern const char version[];
+extern const char version_variant[];
+extern int disableConsoleOutput;
+
+#if __ARM_PAN_AVAILABLE__
+SECURITY_READ_ONLY_LATE(boolean_t) arm_pan_enabled = FALSE; /* PAN support on Hurricane and newer HW */
+#endif
+
+int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
+int pc_trace_cnt = PC_TRACE_BUF_SIZE;
+int debug_task;
+
+boolean_t up_style_idle_exit = 0;
+
+
+
+#if INTERRUPT_MASKED_DEBUG
+boolean_t interrupt_masked_debug = 1;
+uint64_t interrupt_masked_timeout = 0xd0000;
+#endif
+
+boot_args const_boot_args __attribute__((section("__DATA, __const")));
+boot_args *BootArgs __attribute__((section("__DATA, __const")));
+
+unsigned int arm_diag;
+#ifdef APPLETYPHOON
+static unsigned cpus_defeatures = 0x0;
+extern void cpu_defeatures_set(unsigned int);
+#endif
+
+#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
+extern volatile boolean_t arm64_stall_sleep;
+#endif
+
+extern boolean_t force_immediate_debug_halt;
+
+#define MIN_LOW_GLO_MASK (0x144)
+
+/*
+ * Forward definition
+ */
+void arm_init(boot_args * args);
+
+#if __arm64__
+unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
+#endif /* __arm64__ */
+
+
+/*
+ * Routine: arm_init
+ * Function:
+ */
+void
+arm_init(
+ boot_args *args)
+{
+ unsigned int maxmem;
+ uint32_t memsize;
+ uint64_t xmaxmem;
+ thread_t thread;
+ processor_t my_master_proc;
+
+ /* If kernel integrity is supported, use a constant copy of the boot args. */
+ const_boot_args = *args;
+ BootArgs = &const_boot_args;
+
+ cpu_data_init(&BootCpuData);
+
+ PE_init_platform(FALSE, args); /* Get platform expert set up */
+
+#if __arm64__
+ {
+ unsigned int tmp_16k = 0;
+
+#ifdef XXXX
+ /*
+ * Select the advertised kernel page size; without the boot-arg
+ * we default to the hardware page size for the current platform.
+ */
+ if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k)))
+ PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
+ else
+ PAGE_SHIFT_CONST = ARM_PGSHIFT;
+#else
+ /*
+ * Select the advertised kernel page size; with the boot-arg
+ * use to the hardware page size for the current platform.
+ */
+ int radar_20804515 = 1; /* default: new mode */
+ PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515));
+ if (radar_20804515) {
+ if (args->memSize > 1ULL*1024*1024*1024) {
+ /*
+ * arm64 device with > 1GB of RAM:
+ * kernel uses 16KB pages.
+ */
+ PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
+ } else {
+ /*
+ * arm64 device with <= 1GB of RAM:
+ * kernel uses hardware page size
+ * (4KB for H6/H7, 16KB for H8+).
+ */
+ PAGE_SHIFT_CONST = ARM_PGSHIFT;
+ }
+ /* 32-bit apps always see 16KB page size */
+ page_shift_user32 = PAGE_MAX_SHIFT;
+ } else {
+ /* kernel page size: */
+ if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k)))
+ PAGE_SHIFT_CONST = ARM_PGSHIFT;
+ else
+ PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
+ /* old mode: 32-bit apps see same page size as kernel */
+ page_shift_user32 = PAGE_SHIFT_CONST;
+ }
+#endif
+#ifdef APPLETYPHOON
+ if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
+ if ((cpus_defeatures & 0xF) != 0)
+ cpu_defeatures_set(cpus_defeatures & 0xF);
+ }
+#endif
+ }
+#endif
+
+ ml_parse_cpu_topology();
+
+ master_cpu = ml_get_boot_cpu_number();
+ assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
+
+ BootCpuData.cpu_number = (unsigned short)master_cpu;
+#if __arm__
+ BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
+#endif
+ BootCpuData.intstack_top = (vm_offset_t) & intstack_top;
+ BootCpuData.istackptr = BootCpuData.intstack_top;
+ BootCpuData.fiqstack_top = (vm_offset_t) & fiqstack_top;
+ BootCpuData.fiqstackptr = BootCpuData.fiqstack_top;
+#if __arm64__
+ BootCpuData.excepstack_top = (vm_offset_t) & excepstack_top;
+ BootCpuData.excepstackptr = BootCpuData.excepstack_top;
+#endif
+ BootCpuData.cpu_processor = cpu_processor_alloc(TRUE);
+ BootCpuData.cpu_console_buf = (void *)NULL;
+ CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
+ CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
+ + ((uintptr_t)&BootCpuData
+ - (uintptr_t)(args->virtBase)));
+
+ thread_bootstrap();
+ thread = current_thread();
+ /*
+ * Preemption is enabled for this thread so that it can lock mutexes without
+ * tripping the preemption check. In reality scheduling is not enabled until
+ * this thread completes, and there are no other threads to switch to, so
+ * preemption level is not really meaningful for the bootstrap thread.
+ */
+ thread->machine.preemption_count = 0;
+ thread->machine.CpuDatap = &BootCpuData;
+#if __arm__ && __ARM_USER_PROTECT__
+ {
+ unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
+ __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
+ __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
+ __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
+ thread->machine.uptw_ttb = ttbr0_val;
+ thread->machine.kptw_ttb = ttbr1_val;
+ thread->machine.uptw_ttc = ttbcr_val;
+ }
+#endif
+ BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer;
+ BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer;
+
+ cpu_bootstrap();
+
+ rtclock_early_init();
+
+ kernel_early_bootstrap();
+
+ cpu_init();
+
+ EntropyData.index_ptr = EntropyData.buffer;
+
+ processor_bootstrap();
+ my_master_proc = master_processor;
+
+ (void)PE_parse_boot_argn("diag", &arm_diag, sizeof (arm_diag));
+
+ if (PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem)))
+ xmaxmem = (uint64_t) maxmem *(1024 * 1024);
+ else if (PE_get_default("hw.memsize", &memsize, sizeof (memsize)))
+ xmaxmem = (uint64_t) memsize;
+ else
+ xmaxmem = 0;
+
+ if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) {
+ up_style_idle_exit = 1;
+ }
+#if INTERRUPT_MASKED_DEBUG
+ int wdt_boot_arg = 0;
+ /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */
+ if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug,
+ sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg,
+ sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1))) {
+ interrupt_masked_debug = 0;
+ }
+
+ PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout));
+#endif
+
+
+
+ PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
+
+#if __ARM_PAN_AVAILABLE__
+#if (DEVELOPMENT || DEBUG)
+ boolean_t pan;
+ if (!PE_parse_boot_argn("-pmap_smap_disable", &pan, sizeof(pan))) {
+ arm_pan_enabled = TRUE;
+ __builtin_arm_wsr("pan", 1);
+ set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED);
+ }
+#else
+ arm_pan_enabled = TRUE;
+ __builtin_arm_wsr("pan", 1);
+ /* SCTLR_EL1.SPAN is clear on RELEASE */
+#endif
+#endif /* __ARM_PAN_AVAILABLE__ */
+
+ arm_vm_init(xmaxmem, args);
+
+ uint32_t debugmode;
+ if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) &&
+ ((debugmode & MIN_LOW_GLO_MASK) == MIN_LOW_GLO_MASK))
+ patch_low_glo();
+
+ printf_init();
+ panic_init();
+#if __arm64__ && WITH_CLASSIC_S2R
+ sleep_token_buffer_init();
+#endif
+
+ PE_consistent_debug_inherit();
+
+ /* setup debugging output if one has been chosen */
+ PE_init_kprintf(FALSE);
+
+ kprintf("kprintf initialized\n");
+
+ serialmode = 0; /* Assume normal keyboard and console */
+ if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial
+ * keyboard and/or
+ * console? */
+ kprintf("Serial mode specified: %08X\n", serialmode);
+ int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
+ if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
+ if (force_sync) {
+ serialmode |= SERIALMODE_SYNCDRAIN;
+ kprintf(
+ "WARNING: Forcing uart driver to output synchronously."
+ "printf()s/IOLogs will impact kernel performance.\n"
+ "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
+ }
+ }
+ }
+ if (kern_feature_override(KF_SERIAL_OVRD)) {
+ serialmode = 0;
+ }
+
+ if (serialmode & SERIALMODE_OUTPUT) { /* Start serial if requested */
+ (void)switch_to_serial_console(); /* Switch into serial mode */
+ disableConsoleOutput = FALSE; /* Allow printfs to happen */
+ }
+ PE_create_console();
+
+ /* setup console output */
+ PE_init_printf(FALSE);
+
+#if __arm64__
+#if DEBUG
+ dump_kva_space();
+#endif
+#endif
+
+ cpu_machine_idle_init(TRUE);
+
+#if (__ARM_ARCH__ == 7)
+ if (arm_diag & 0x8000)
+ set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
+#endif
+
+ PE_init_platform(TRUE, &BootCpuData);
+ cpu_timebase_init(TRUE);
+ fiq_context_init(TRUE);
+
+
+ /*
+ * Initialize the stack protector for all future calls
+ * to C code. Since kernel_bootstrap() eventually
+ * switches stack context without returning through this
+ * function, we do not risk failing the check even though
+ * we mutate the guard word during execution.
+ */
+ __stack_chk_guard = (unsigned long)early_random();
+ /* Zero a byte of the protector to guard
+ * against string vulnerabilities
+ */
+ __stack_chk_guard &= ~(0xFFULL << 8);
+ machine_startup(args);
+}
+
+/*
+ * Routine: arm_init_cpu
+ * Function:
+ * Re-initialize CPU when coming out of reset
+ */
+
+void
+arm_init_cpu(
+ cpu_data_t *cpu_data_ptr)
+{
+#if __ARM_PAN_AVAILABLE__
+#if (DEVELOPMENT || DEBUG)
+ if (arm_pan_enabled) {
+ __builtin_arm_wsr("pan", 1);
+ set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED);
+ }
+#else
+ __builtin_arm_wsr("pan", 1);
+ /* SCTLR_EL1.SPAN is clear on RELEASE */
+#endif
+#endif
+
+ cpu_data_ptr->cpu_flags &= ~SleepState;
+#if __ARM_SMP__ && defined(ARMA7)
+ cpu_data_ptr->cpu_CLW_active = 1;
+#endif
+
+ machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
+
+#if __arm64__
+ /* Enable asynchronous exceptions */
+ __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
+#endif
+
+ cpu_machine_idle_init(FALSE);
+
+ cpu_init();
+
+#if (__ARM_ARCH__ == 7)
+ if (arm_diag & 0x8000)
+ set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
+#endif
+#ifdef APPLETYPHOON
+ if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0)
+ cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF);
+#endif
+ /* Initialize the timebase before serial_init, as some serial
+ * drivers use mach_absolute_time() to implement rate control
+ */
+ cpu_timebase_init(FALSE);
+
+ if (cpu_data_ptr == &BootCpuData) {
+#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
+ /*
+ * Prevent CPUs from going into deep sleep until all
+ * CPUs are ready to do so.
+ */
+ arm64_stall_sleep = TRUE;
+#endif
+ serial_init();
+ PE_init_platform(TRUE, NULL);
+ commpage_update_timebase();
+ }
+
+ fiq_context_init(TRUE);
+ cpu_data_ptr->rtcPop = EndOfAllTime;
+ timer_resync_deadlines();
+
+#if DEVELOPMENT || DEBUG
+ PE_arm_debug_enable_trace();
+#endif
+
+ kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id);
+
+ if (cpu_data_ptr == &BootCpuData) {
+#if CONFIG_TELEMETRY
+ bootprofile_wake_from_sleep();
+#endif /* CONFIG_TELEMETRY */
+#if MONOTONIC && defined(__arm64__)
+ mt_wake();
+#endif /* MONOTONIC && defined(__arm64__) */
+ }
+
+ slave_main(NULL);
+}
+
+/*
+ * Routine: arm_init_idle_cpu
+ * Function:
+ */
+void __attribute__((noreturn))
+arm_init_idle_cpu(
+ cpu_data_t *cpu_data_ptr)
+{
+#if __ARM_PAN_AVAILABLE__
+#if (DEVELOPMENT || DEBUG)
+ if (arm_pan_enabled) {
+ __builtin_arm_wsr("pan", 1);
+ set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED);
+ }
+#else
+ __builtin_arm_wsr("pan", 1);
+ /* SCTLR_EL1.SPAN is clear on RELEASE */
+#endif
+#endif
+#if __ARM_SMP__ && defined(ARMA7)
+ cpu_data_ptr->cpu_CLW_active = 1;
+#endif
+
+ machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
+
+#if __arm64__
+ /* Enable asynchronous exceptions */
+ __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
+#endif
+
+#if (__ARM_ARCH__ == 7)
+ if (arm_diag & 0x8000)
+ set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
+#endif
+#ifdef APPLETYPHOON
+ if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0)
+ cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF);
+#endif
+
+ fiq_context_init(FALSE);
+
+ cpu_idle_exit();
+}
projects / apple / xnu.git / blobdiff