X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4d15aeb193b2c68f1d38666c317f8d3734f5f083..5ba3f43ea354af8ad55bea84372a2bc834d8757c:/osfmk/arm/arm_init.c?ds=sidebyside diff --git a/osfmk/arm/arm_init.c b/osfmk/arm/arm_init.c new file mode 100644 index 000000000..a9bb6d407 --- /dev/null +++ b/osfmk/arm/arm_init.c @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* ARM64_TODO unify boot.h */ +#if __arm64__ +#include +#elif __arm__ +#include +#else +#error Unsupported arch +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#if CONFIG_TELEMETRY +#include +#endif +#if MONOTONIC +#include +#endif /* MONOTONIC */ + +extern void patch_low_glo(void); +extern int serial_init(void); +extern void sleep_token_buffer_init(void); + +extern vm_offset_t intstack_top; +extern vm_offset_t fiqstack_top; +#if __arm64__ +extern vm_offset_t excepstack_top; +#endif + +extern const char version[]; +extern const char version_variant[]; +extern int disableConsoleOutput; + +#if __ARM_PAN_AVAILABLE__ +SECURITY_READ_ONLY_LATE(boolean_t) arm_pan_enabled = FALSE; /* PAN support on Hurricane and newer HW */ +#endif + +int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0}; +int pc_trace_cnt = PC_TRACE_BUF_SIZE; +int debug_task; + +boolean_t up_style_idle_exit = 0; + + + +#if INTERRUPT_MASKED_DEBUG +boolean_t interrupt_masked_debug = 1; +uint64_t interrupt_masked_timeout = 0xd0000; +#endif + +boot_args const_boot_args __attribute__((section("__DATA, __const"))); +boot_args *BootArgs __attribute__((section("__DATA, __const"))); + +unsigned int arm_diag; +#ifdef APPLETYPHOON +static unsigned cpus_defeatures = 0x0; +extern void cpu_defeatures_set(unsigned int); +#endif + +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ +extern volatile boolean_t arm64_stall_sleep; +#endif + +extern boolean_t force_immediate_debug_halt; + +#define MIN_LOW_GLO_MASK (0x144) + +/* + * Forward definition + */ +void arm_init(boot_args * args); + +#if __arm64__ +unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */ +#endif /* __arm64__ */ + + +/* + * Routine: arm_init + * Function: + */ +void +arm_init( + boot_args *args) +{ + unsigned int maxmem; + uint32_t memsize; + uint64_t xmaxmem; + thread_t thread; + processor_t my_master_proc; + + /* If kernel integrity is supported, use a constant copy of the boot args. */ + const_boot_args = *args; + BootArgs = &const_boot_args; + + cpu_data_init(&BootCpuData); + + PE_init_platform(FALSE, args); /* Get platform expert set up */ + +#if __arm64__ + { + unsigned int tmp_16k = 0; + +#ifdef XXXX + /* + * Select the advertised kernel page size; without the boot-arg + * we default to the hardware page size for the current platform. + */ + if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + else + PAGE_SHIFT_CONST = ARM_PGSHIFT; +#else + /* + * Select the advertised kernel page size; with the boot-arg + * use to the hardware page size for the current platform. + */ + int radar_20804515 = 1; /* default: new mode */ + PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515)); + if (radar_20804515) { + if (args->memSize > 1ULL*1024*1024*1024) { + /* + * arm64 device with > 1GB of RAM: + * kernel uses 16KB pages. + */ + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + } else { + /* + * arm64 device with <= 1GB of RAM: + * kernel uses hardware page size + * (4KB for H6/H7, 16KB for H8+). + */ + PAGE_SHIFT_CONST = ARM_PGSHIFT; + } + /* 32-bit apps always see 16KB page size */ + page_shift_user32 = PAGE_MAX_SHIFT; + } else { + /* kernel page size: */ + if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = ARM_PGSHIFT; + else + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + /* old mode: 32-bit apps see same page size as kernel */ + page_shift_user32 = PAGE_SHIFT_CONST; + } +#endif +#ifdef APPLETYPHOON + if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) { + if ((cpus_defeatures & 0xF) != 0) + cpu_defeatures_set(cpus_defeatures & 0xF); + } +#endif + } +#endif + + ml_parse_cpu_topology(); + + master_cpu = ml_get_boot_cpu_number(); + assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number()); + + BootCpuData.cpu_number = (unsigned short)master_cpu; +#if __arm__ + BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable; +#endif + BootCpuData.intstack_top = (vm_offset_t) & intstack_top; + BootCpuData.istackptr = BootCpuData.intstack_top; + BootCpuData.fiqstack_top = (vm_offset_t) & fiqstack_top; + BootCpuData.fiqstackptr = BootCpuData.fiqstack_top; +#if __arm64__ + BootCpuData.excepstack_top = (vm_offset_t) & excepstack_top; + BootCpuData.excepstackptr = BootCpuData.excepstack_top; +#endif + BootCpuData.cpu_processor = cpu_processor_alloc(TRUE); + BootCpuData.cpu_console_buf = (void *)NULL; + CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData; + CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase) + + ((uintptr_t)&BootCpuData + - (uintptr_t)(args->virtBase))); + + thread_bootstrap(); + thread = current_thread(); + /* + * Preemption is enabled for this thread so that it can lock mutexes without + * tripping the preemption check. In reality scheduling is not enabled until + * this thread completes, and there are no other threads to switch to, so + * preemption level is not really meaningful for the bootstrap thread. + */ + thread->machine.preemption_count = 0; + thread->machine.CpuDatap = &BootCpuData; +#if __arm__ && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val, ttbcr_val; + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val)); + thread->machine.uptw_ttb = ttbr0_val; + thread->machine.kptw_ttb = ttbr1_val; + thread->machine.uptw_ttc = ttbcr_val; + } +#endif + BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer; + BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer; + + cpu_bootstrap(); + + rtclock_early_init(); + + kernel_early_bootstrap(); + + cpu_init(); + + EntropyData.index_ptr = EntropyData.buffer; + + processor_bootstrap(); + my_master_proc = master_processor; + + (void)PE_parse_boot_argn("diag", &arm_diag, sizeof (arm_diag)); + + if (PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem))) + xmaxmem = (uint64_t) maxmem *(1024 * 1024); + else if (PE_get_default("hw.memsize", &memsize, sizeof (memsize))) + xmaxmem = (uint64_t) memsize; + else + xmaxmem = 0; + + if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) { + up_style_idle_exit = 1; + } +#if INTERRUPT_MASKED_DEBUG + int wdt_boot_arg = 0; + /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */ + if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug, + sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg, + sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1))) { + interrupt_masked_debug = 0; + } + + PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout)); +#endif + + + + PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt)); + +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + boolean_t pan; + if (!PE_parse_boot_argn("-pmap_smap_disable", &pan, sizeof(pan))) { + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif /* __ARM_PAN_AVAILABLE__ */ + + arm_vm_init(xmaxmem, args); + + uint32_t debugmode; + if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) && + ((debugmode & MIN_LOW_GLO_MASK) == MIN_LOW_GLO_MASK)) + patch_low_glo(); + + printf_init(); + panic_init(); +#if __arm64__ && WITH_CLASSIC_S2R + sleep_token_buffer_init(); +#endif + + PE_consistent_debug_inherit(); + + /* setup debugging output if one has been chosen */ + PE_init_kprintf(FALSE); + + kprintf("kprintf initialized\n"); + + serialmode = 0; /* Assume normal keyboard and console */ + if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial + * keyboard and/or + * console? */ + kprintf("Serial mode specified: %08X\n", serialmode); + int force_sync = serialmode & SERIALMODE_SYNCDRAIN; + if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) { + if (force_sync) { + serialmode |= SERIALMODE_SYNCDRAIN; + kprintf( + "WARNING: Forcing uart driver to output synchronously." + "printf()s/IOLogs will impact kernel performance.\n" + "You are advised to avoid using 'drain_uart_sync' boot-arg.\n"); + } + } + } + if (kern_feature_override(KF_SERIAL_OVRD)) { + serialmode = 0; + } + + if (serialmode & SERIALMODE_OUTPUT) { /* Start serial if requested */ + (void)switch_to_serial_console(); /* Switch into serial mode */ + disableConsoleOutput = FALSE; /* Allow printfs to happen */ + } + PE_create_console(); + + /* setup console output */ + PE_init_printf(FALSE); + +#if __arm64__ +#if DEBUG + dump_kva_space(); +#endif +#endif + + cpu_machine_idle_init(TRUE); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif + + PE_init_platform(TRUE, &BootCpuData); + cpu_timebase_init(TRUE); + fiq_context_init(TRUE); + + + /* + * Initialize the stack protector for all future calls + * to C code. Since kernel_bootstrap() eventually + * switches stack context without returning through this + * function, we do not risk failing the check even though + * we mutate the guard word during execution. + */ + __stack_chk_guard = (unsigned long)early_random(); + /* Zero a byte of the protector to guard + * against string vulnerabilities + */ + __stack_chk_guard &= ~(0xFFULL << 8); + machine_startup(args); +} + +/* + * Routine: arm_init_cpu + * Function: + * Re-initialize CPU when coming out of reset + */ + +void +arm_init_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif + + cpu_data_ptr->cpu_flags &= ~SleepState; +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + + cpu_machine_idle_init(FALSE); + + cpu_init(); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + /* Initialize the timebase before serial_init, as some serial + * drivers use mach_absolute_time() to implement rate control + */ + cpu_timebase_init(FALSE); + + if (cpu_data_ptr == &BootCpuData) { +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ + /* + * Prevent CPUs from going into deep sleep until all + * CPUs are ready to do so. + */ + arm64_stall_sleep = TRUE; +#endif + serial_init(); + PE_init_platform(TRUE, NULL); + commpage_update_timebase(); + } + + fiq_context_init(TRUE); + cpu_data_ptr->rtcPop = EndOfAllTime; + timer_resync_deadlines(); + +#if DEVELOPMENT || DEBUG + PE_arm_debug_enable_trace(); +#endif + + kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id); + + if (cpu_data_ptr == &BootCpuData) { +#if CONFIG_TELEMETRY + bootprofile_wake_from_sleep(); +#endif /* CONFIG_TELEMETRY */ +#if MONOTONIC && defined(__arm64__) + mt_wake(); +#endif /* MONOTONIC && defined(__arm64__) */ + } + + slave_main(NULL); +} + +/* + * Routine: arm_init_idle_cpu + * Function: + */ +void __attribute__((noreturn)) +arm_init_idle_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + + fiq_context_init(FALSE); + + cpu_idle_exit(); +}