osfmk/arm/arm_init.c

   1 /*
   2  * Copyright (c) 2007-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 #include <debug.h>
  33 #include <mach_ldebug.h>
  34 #include <mach_kdp.h>
  35
  36 #include <kern/misc_protos.h>
  37 #include <kern/thread.h>
  38 #include <kern/timer_queue.h>
  39 #include <kern/processor.h>
  40 #include <kern/startup.h>
  41 #include <kern/debug.h>
  42 #include <prng/random.h>
  43 #include <machine/machine_routines.h>
  44 #include <machine/commpage.h>
  45 /* ARM64_TODO unify boot.h */
  46 #if __arm64__
  47 #include <pexpert/arm64/boot.h>
  48 #elif __arm__
  49 #include <pexpert/arm/boot.h>
  50 #else
  51 #error Unsupported arch
  52 #endif
  53 #include <pexpert/arm/consistent_debug.h>
  54 #include <pexpert/device_tree.h>
  55 #include <arm/proc_reg.h>
  56 #include <arm/pmap.h>
  57 #include <arm/caches_internal.h>
  58 #include <arm/cpu_internal.h>
  59 #include <arm/cpu_data_internal.h>
  60 #include <arm/misc_protos.h>
  61 #include <arm/machine_cpu.h>
  62 #include <arm/rtclock.h>
  63 #include <vm/vm_map.h>
  64
  65 #include <libkern/kernel_mach_header.h>
  66 #include <libkern/stack_protector.h>
  67 #include <libkern/section_keywords.h>
  68 #include <san/kasan.h>
  69
  70 #include <pexpert/pexpert.h>
  71
  72 #include <console/serial_protos.h>
  73
  74 #if CONFIG_TELEMETRY
  75 #include <kern/telemetry.h>
  76 #endif
  77 #if MONOTONIC
  78 #include <kern/monotonic.h>
  79 #endif /* MONOTONIC */
  80
  81 extern void     patch_low_glo(void);
  82 extern int      serial_init(void);
  83 extern void sleep_token_buffer_init(void);
  84
  85 extern vm_offset_t intstack_top;
  86 extern vm_offset_t fiqstack_top;
  87 #if __arm64__
  88 extern vm_offset_t excepstack_top;
  89 #endif
  90
  91 extern const char version[];
  92 extern const char version_variant[];
  93 extern int      disableConsoleOutput;
  94
  95 int             pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
  96 int             pc_trace_cnt = PC_TRACE_BUF_SIZE;
  97 int             debug_task;
  98
  99 boolean_t up_style_idle_exit = 0;
 100
 101
 102
 103 #if INTERRUPT_MASKED_DEBUG
 104 boolean_t interrupt_masked_debug = 1;
 105 uint64_t interrupt_masked_timeout = 0xd0000;
 106 #endif
 107
 108 boot_args const_boot_args __attribute__((section("__DATA, __const")));
 109 boot_args      *BootArgs __attribute__((section("__DATA, __const")));
 110
 111 unsigned int arm_diag;
 112 #ifdef  APPLETYPHOON
 113 static unsigned cpus_defeatures = 0x0;
 114 extern void cpu_defeatures_set(unsigned int);
 115 #endif
 116
 117 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 118 extern volatile boolean_t arm64_stall_sleep;
 119 #endif
 120
 121 extern boolean_t force_immediate_debug_halt;
 122
 123 #define MIN_LOW_GLO_MASK (0x144)
 124
 125 /*
 126  * Forward definition
 127  */
 128 void arm_init(boot_args * args);
 129
 130 #if __arm64__
 131 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
 132 #endif /* __arm64__ */
 133
 134
 135 /*
 136  *              Routine:                arm_init
 137  *              Function:
 138  */
 139 void
 140 arm_init(
 141         boot_args       *args)
 142 {
 143         unsigned int    maxmem;
 144         uint32_t        memsize;
 145         uint64_t        xmaxmem;
 146         thread_t        thread;
 147         processor_t     my_master_proc;
 148
 149         /* If kernel integrity is supported, use a constant copy of the boot args. */
 150         const_boot_args = *args;
 151         BootArgs = &const_boot_args;
 152
 153         cpu_data_init(&BootCpuData);
 154
 155         PE_init_platform(FALSE, args);  /* Get platform expert set up */
 156
 157 #if __arm64__
 158         {
 159                 unsigned int    tmp_16k = 0;
 160
 161 #ifdef  XXXX
 162                 /*
 163                  * Select the advertised kernel page size; without the boot-arg
 164                  * we default to the hardware page size for the current platform.
 165                  */
 166                 if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k)))
 167                         PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 168                 else
 169                         PAGE_SHIFT_CONST = ARM_PGSHIFT;
 170 #else
 171                 /*
 172                  * Select the advertised kernel page size; with the boot-arg
 173                  * use to the hardware page size for the current platform.
 174                  */
 175                 int radar_20804515 = 1; /* default: new mode */
 176                 PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515));
 177                 if (radar_20804515) {
 178                         if (args->memSize > 1ULL*1024*1024*1024) {
 179                                 /*
 180                                  * arm64 device with > 1GB of RAM:
 181                                  * kernel uses 16KB pages.
 182                                  */
 183                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 184                         } else {
 185                                 /*
 186                                  * arm64 device with <= 1GB of RAM:
 187                                  * kernel uses hardware page size
 188                                  * (4KB for H6/H7, 16KB for H8+).
 189                                  */
 190                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 191                         }
 192                         /* 32-bit apps always see 16KB page size */
 193                         page_shift_user32 = PAGE_MAX_SHIFT;
 194                 } else {
 195                         /* kernel page size: */
 196                         if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k)))
 197                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 198                         else
 199                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 200                         /* old mode: 32-bit apps see same page size as kernel */
 201                         page_shift_user32 = PAGE_SHIFT_CONST;
 202                 }
 203 #endif
 204 #ifdef  APPLETYPHOON
 205                 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
 206                         if ((cpus_defeatures & 0xF) != 0)
 207                                 cpu_defeatures_set(cpus_defeatures & 0xF);
 208                 }
 209 #endif
 210         }
 211 #endif
 212
 213         ml_parse_cpu_topology();
 214
 215         master_cpu = ml_get_boot_cpu_number();
 216         assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
 217
 218         BootCpuData.cpu_number = (unsigned short)master_cpu;
 219 #if     __arm__
 220         BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 221 #endif
 222         BootCpuData.intstack_top = (vm_offset_t) & intstack_top;
 223         BootCpuData.istackptr = BootCpuData.intstack_top;
 224         BootCpuData.fiqstack_top = (vm_offset_t) & fiqstack_top;
 225         BootCpuData.fiqstackptr = BootCpuData.fiqstack_top;
 226 #if __arm64__
 227         BootCpuData.excepstack_top = (vm_offset_t) & excepstack_top;
 228         BootCpuData.excepstackptr = BootCpuData.excepstack_top;
 229 #endif
 230         BootCpuData.cpu_processor = cpu_processor_alloc(TRUE);
 231         BootCpuData.cpu_console_buf = (void *)NULL;
 232         CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
 233         CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
 234                                                     + ((uintptr_t)&BootCpuData
 235                                                     - (uintptr_t)(args->virtBase)));
 236
 237         thread_bootstrap();
 238         thread = current_thread();
 239         /*
 240          * Preemption is enabled for this thread so that it can lock mutexes without
 241          * tripping the preemption check. In reality scheduling is not enabled until
 242          * this thread completes, and there are no other threads to switch to, so
 243          * preemption level is not really meaningful for the bootstrap thread.
 244          */
 245         thread->machine.preemption_count = 0;
 246         thread->machine.CpuDatap = &BootCpuData;
 247 #if     __arm__ && __ARM_USER_PROTECT__
 248         {
 249                 unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
 250                 __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
 251                 __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
 252                 __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
 253                 thread->machine.uptw_ttb = ttbr0_val;
 254                 thread->machine.kptw_ttb = ttbr1_val;
 255                 thread->machine.uptw_ttc = ttbcr_val;
 256         }
 257 #endif
 258         BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer;
 259         BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer;
 260
 261         cpu_bootstrap();
 262
 263         rtclock_early_init();
 264
 265         kernel_early_bootstrap();
 266
 267         cpu_init();
 268
 269         EntropyData.index_ptr = EntropyData.buffer;
 270
 271         processor_bootstrap();
 272         my_master_proc = master_processor;
 273
 274         (void)PE_parse_boot_argn("diag", &arm_diag, sizeof (arm_diag));
 275
 276         if (PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem)))
 277                 xmaxmem = (uint64_t) maxmem *(1024 * 1024);
 278         else if (PE_get_default("hw.memsize", &memsize, sizeof (memsize)))
 279                 xmaxmem = (uint64_t) memsize;
 280         else
 281                 xmaxmem = 0;
 282
 283         if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) {
 284                 up_style_idle_exit = 1;
 285         }
 286 #if INTERRUPT_MASKED_DEBUG
 287         int wdt_boot_arg = 0;
 288         /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */
 289         if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug,
 290                                 sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg,
 291                                 sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1)))  {
 292                 interrupt_masked_debug = 0;
 293         }
 294
 295         PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout));
 296 #endif
 297
 298
 299
 300         PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
 301
 302 #if __ARM_PAN_AVAILABLE__
 303         __builtin_arm_wsr("pan", 1);
 304 #endif  /* __ARM_PAN_AVAILABLE__ */
 305
 306         arm_vm_init(xmaxmem, args);
 307
 308         uint32_t debugmode;
 309         if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) &&
 310            ((debugmode & MIN_LOW_GLO_MASK) == MIN_LOW_GLO_MASK))
 311                 patch_low_glo();
 312
 313         printf_init();
 314         panic_init();
 315 #if __arm64__ && WITH_CLASSIC_S2R
 316         sleep_token_buffer_init();
 317 #endif
 318
 319         PE_consistent_debug_inherit();
 320
 321         /* setup debugging output if one has been chosen */
 322         PE_init_kprintf(FALSE);
 323
 324         kprintf("kprintf initialized\n");
 325
 326         serialmode = 0;                                                      /* Assume normal keyboard and console */
 327         if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial
 328                                                                               * keyboard and/or
 329                                                                               * console? */
 330                 kprintf("Serial mode specified: %08X\n", serialmode);
 331                 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
 332                 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
 333                         if (force_sync) {
 334                                 serialmode |= SERIALMODE_SYNCDRAIN;
 335                                 kprintf(
 336                                     "WARNING: Forcing uart driver to output synchronously."
 337                                     "printf()s/IOLogs will impact kernel performance.\n"
 338                                     "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
 339                         }
 340                 }
 341         }
 342         if (kern_feature_override(KF_SERIAL_OVRD)) {
 343                 serialmode = 0;
 344         }
 345
 346         if (serialmode & SERIALMODE_OUTPUT) {                 /* Start serial if requested */
 347                 (void)switch_to_serial_console(); /* Switch into serial mode */
 348                 disableConsoleOutput = FALSE;     /* Allow printfs to happen */
 349         }
 350         PE_create_console();
 351
 352         /* setup console output */
 353         PE_init_printf(FALSE);
 354
 355 #if __arm64__
 356 #if DEBUG
 357         dump_kva_space();
 358 #endif
 359 #endif
 360
 361         cpu_machine_idle_init(TRUE);
 362
 363 #if     (__ARM_ARCH__ == 7)
 364         if (arm_diag & 0x8000)
 365                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 366 #endif
 367
 368         PE_init_platform(TRUE, &BootCpuData);
 369         cpu_timebase_init(TRUE);
 370         fiq_context_init(TRUE);
 371
 372
 373         /*
 374          * Initialize the stack protector for all future calls
 375          * to C code. Since kernel_bootstrap() eventually
 376          * switches stack context without returning through this
 377          * function, we do not risk failing the check even though
 378          * we mutate the guard word during execution.
 379          */
 380         __stack_chk_guard = (unsigned long)early_random();
 381         /* Zero a byte of the protector to guard
 382          * against string vulnerabilities
 383          */
 384         __stack_chk_guard &= ~(0xFFULL << 8);
 385         machine_startup(args);
 386 }
 387
 388 /*
 389  * Routine:        arm_init_cpu
 390  * Function:
 391  *    Re-initialize CPU when coming out of reset
 392  */
 393
 394 void
 395 arm_init_cpu(
 396         cpu_data_t      *cpu_data_ptr)
 397 {
 398 #if __ARM_PAN_AVAILABLE__
 399         __builtin_arm_wsr("pan", 1);
 400 #endif
 401
 402         cpu_data_ptr->cpu_flags &= ~SleepState;
 403 #if     __ARM_SMP__ && defined(ARMA7)
 404         cpu_data_ptr->cpu_CLW_active = 1;
 405 #endif
 406
 407         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 408
 409 #if __arm64__
 410         /* Enable asynchronous exceptions */
 411         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 412 #endif
 413
 414         cpu_machine_idle_init(FALSE);
 415
 416         cpu_init();
 417
 418 #if     (__ARM_ARCH__ == 7)
 419         if (arm_diag & 0x8000)
 420                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 421 #endif
 422 #ifdef  APPLETYPHOON
 423         if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0)
 424                 cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF);
 425 #endif
 426         /* Initialize the timebase before serial_init, as some serial
 427          * drivers use mach_absolute_time() to implement rate control
 428          */
 429         cpu_timebase_init(FALSE);
 430
 431         if (cpu_data_ptr == &BootCpuData) {
 432 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 433                 /*
 434                  * Prevent CPUs from going into deep sleep until all
 435                  * CPUs are ready to do so.
 436                  */
 437                 arm64_stall_sleep = TRUE;
 438 #endif
 439                 serial_init();
 440                 PE_init_platform(TRUE, NULL);
 441                 commpage_update_timebase();
 442         }
 443
 444         fiq_context_init(TRUE);
 445         cpu_data_ptr->rtcPop = EndOfAllTime;
 446         timer_resync_deadlines();
 447
 448 #if DEVELOPMENT || DEBUG
 449         PE_arm_debug_enable_trace();
 450 #endif
 451
 452         kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id);
 453
 454         if (cpu_data_ptr == &BootCpuData) {
 455 #if CONFIG_TELEMETRY
 456                 bootprofile_wake_from_sleep();
 457 #endif /* CONFIG_TELEMETRY */
 458 #if MONOTONIC && defined(__arm64__)
 459                 mt_wake();
 460 #endif /* MONOTONIC && defined(__arm64__) */
 461         }
 462
 463         slave_main(NULL);
 464 }
 465
 466 /*
 467  * Routine:        arm_init_idle_cpu
 468  * Function:
 469  */
 470 void __attribute__((noreturn))
 471 arm_init_idle_cpu(
 472         cpu_data_t      *cpu_data_ptr)
 473 {
 474 #if __ARM_PAN_AVAILABLE__
 475         __builtin_arm_wsr("pan", 1);
 476 #endif
 477 #if     __ARM_SMP__ && defined(ARMA7)
 478         cpu_data_ptr->cpu_CLW_active = 1;
 479 #endif
 480
 481         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 482
 483 #if __arm64__
 484         /* Enable asynchronous exceptions */
 485         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 486 #endif
 487
 488 #if     (__ARM_ARCH__ == 7)
 489         if (arm_diag & 0x8000)
 490                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 491 #endif
 492 #ifdef  APPLETYPHOON
 493         if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0)
 494                 cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF);
 495 #endif
 496
 497         fiq_context_init(FALSE);
 498
 499         cpu_idle_exit();
 500 }