osfmk/arm/arm_init.c

   1 /*
   2  * Copyright (c) 2007-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 #include <debug.h>
  33 #include <mach_ldebug.h>
  34 #include <mach_kdp.h>
  35
  36 #include <kern/misc_protos.h>
  37 #include <kern/thread.h>
  38 #include <kern/timer_queue.h>
  39 #include <kern/processor.h>
  40 #include <kern/startup.h>
  41 #include <kern/debug.h>
  42 #include <prng/random.h>
  43 #include <machine/machine_routines.h>
  44 #include <machine/commpage.h>
  45 #if HIBERNATION
  46 #include <machine/pal_hibernate.h>
  47 #endif /* HIBERNATION */
  48 /* ARM64_TODO unify boot.h */
  49 #if __arm64__
  50 #include <pexpert/arm64/apple_arm64_common.h>
  51 #include <pexpert/arm64/boot.h>
  52 #elif __arm__
  53 #include <pexpert/arm/boot.h>
  54 #else
  55 #error Unsupported arch
  56 #endif
  57 #include <pexpert/arm/consistent_debug.h>
  58 #include <pexpert/device_tree.h>
  59 #include <arm/proc_reg.h>
  60 #include <arm/pmap.h>
  61 #include <arm/caches_internal.h>
  62 #include <arm/cpu_internal.h>
  63 #include <arm/cpu_data_internal.h>
  64 #include <arm/cpuid_internal.h>
  65 #include <arm/io_map_entries.h>
  66 #include <arm/misc_protos.h>
  67 #include <arm/machine_cpu.h>
  68 #include <arm/rtclock.h>
  69 #include <vm/vm_map.h>
  70
  71 #include <libkern/kernel_mach_header.h>
  72 #include <libkern/stack_protector.h>
  73 #include <libkern/section_keywords.h>
  74 #include <san/kasan.h>
  75 #include <sys/kdebug.h>
  76
  77 #include <pexpert/pexpert.h>
  78
  79 #include <console/serial_protos.h>
  80
  81 #if CONFIG_TELEMETRY
  82 #include <kern/telemetry.h>
  83 #endif
  84 #if MONOTONIC
  85 #include <kern/monotonic.h>
  86 #endif /* MONOTONIC */
  87
  88 #if HIBERNATION
  89 #include <IOKit/IOPlatformExpert.h>
  90 #endif /* HIBERNATION */
  91
  92 extern void     patch_low_glo(void);
  93 extern int      serial_init(void);
  94 extern void sleep_token_buffer_init(void);
  95
  96 extern vm_offset_t intstack_top;
  97 #if __arm64__
  98 extern vm_offset_t excepstack_top;
  99 #else
 100 extern vm_offset_t fiqstack_top;
 101 #endif
 102
 103 extern const char version[];
 104 extern const char version_variant[];
 105 extern int      disableConsoleOutput;
 106
 107 int             pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
 108 int             pc_trace_cnt = PC_TRACE_BUF_SIZE;
 109 int             debug_task;
 110
 111 bool need_wa_rdar_55577508 = false;
 112 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
 113
 114
 115 #if HAS_BP_RET
 116 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
 117 uint32_t bp_ret = 3;
 118 extern void set_bp_ret(void);
 119 #endif
 120
 121 #if INTERRUPT_MASKED_DEBUG
 122 boolean_t interrupt_masked_debug = 1;
 123 /* the following are in mach timebase units */
 124 uint64_t interrupt_masked_timeout = 0xd0000;
 125 uint64_t stackshot_interrupt_masked_timeout = 0xf9999;
 126 #endif
 127
 128 boot_args const_boot_args __attribute__((section("__DATA, __const")));
 129 boot_args      *BootArgs __attribute__((section("__DATA, __const")));
 130
 131 TUNABLE(uint32_t, arm_diag, "diag", 0);
 132 #ifdef  APPLETYPHOON
 133 static unsigned cpus_defeatures = 0x0;
 134 extern void cpu_defeatures_set(unsigned int);
 135 #endif
 136
 137 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 138 extern volatile boolean_t arm64_stall_sleep;
 139 #endif
 140
 141 extern boolean_t force_immediate_debug_halt;
 142
 143 #if HAS_APPLE_PAC
 144 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
 145 #endif
 146
 147 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
 148 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
 149
 150 /*
 151  * Forward definition
 152  */
 153 void arm_init(boot_args * args);
 154
 155 #if __arm64__
 156 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
 157 #endif /* __arm64__ */
 158
 159
 160 /*
 161  * JOP rebasing
 162  */
 163
 164 #define dyldLogFunc(msg, ...)
 165 #include <mach/dyld_kernel_fixups.h>
 166
 167 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
 168 extern uint32_t __thread_starts_sect_end[]   __asm("section$end$__TEXT$__thread_starts");
 169 #if defined(HAS_APPLE_PAC)
 170 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
 171 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
 172 #endif /* defined(HAS_APPLE_PAC) */
 173
 174 extern vm_offset_t vm_kernel_slide;
 175 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
 176 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
 177 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
 178
 179 static void
 180 arm_slide_rebase_and_sign_image(void)
 181 {
 182         kernel_mach_header_t *k_mh, *kc_mh = NULL;
 183         kernel_segment_command_t *seg;
 184         uintptr_t slide;
 185
 186         k_mh = &_mh_execute_header;
 187         if (kernel_mach_header_is_in_fileset(k_mh)) {
 188                 /*
 189                  * The kernel is part of a MH_FILESET kernel collection, determine slide
 190                  * based on first segment's mach-o vmaddr (requires first kernel load
 191                  * command to be LC_SEGMENT_64 of the __TEXT segment)
 192                  */
 193                 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
 194                 assert(seg->cmd == LC_SEGMENT_KERNEL);
 195                 slide = (uintptr_t)k_mh - seg->vmaddr;
 196
 197                 /*
 198                  * The kernel collection linker guarantees that the boot collection mach
 199                  * header vmaddr is the hardcoded kernel link address (as specified to
 200                  * ld64 when linking the kernel).
 201                  */
 202                 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
 203                 assert(kc_mh->filetype == MH_FILESET);
 204
 205                 /*
 206                  * rebase and sign jops
 207                  * Note that we can't call any functions before this point, so
 208                  * we have to hard-code the knowledge that the base of the KC
 209                  * is the KC's mach-o header. This would change if any
 210                  * segment's VA started *before* the text segment
 211                  * (as the HIB segment does on x86).
 212                  */
 213                 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
 214                 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
 215
 216                 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
 217
 218                 /*
 219                  * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
 220                  * collection, so adjust them now, and determine the vmaddr range
 221                  * covered by read-only segments for the CTRR rorgn.
 222                  */
 223                 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
 224                     (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
 225                     (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
 226                     NULL, NULL, NULL);
 227 #if defined(HAS_APPLE_PAC)
 228                 OSRuntimeSignStructorsInFileset(kc_mh);
 229 #endif /* defined(HAS_APPLE_PAC) */
 230         } else {
 231                 /*
 232                  * Static kernelcache: iBoot slid kernel MachO vmaddrs, determine slide
 233                  * using hardcoded kernel link address
 234                  */
 235                 slide = (uintptr_t)k_mh - VM_KERNEL_LINK_ADDRESS;
 236
 237                 /* rebase and sign jops */
 238                 static_kernelcache = &__thread_starts_sect_end[0] != &__thread_starts_sect_start[0];
 239                 if (static_kernelcache) {
 240                         rebase_threaded_starts( &__thread_starts_sect_start[0],
 241                             &__thread_starts_sect_end[0],
 242                             (uintptr_t)k_mh, (uintptr_t)k_mh - slide, slide);
 243                 }
 244 #if defined(HAS_APPLE_PAC)
 245                 OSRuntimeSignStructors(&_mh_execute_header);
 246 #endif /* defined(HAS_APPLE_PAC) */
 247         }
 248
 249
 250         /*
 251          * Initialize slide global here to avoid duplicating this logic in
 252          * arm_vm_init()
 253          */
 254         vm_kernel_slide = slide;
 255 }
 256
 257 void
 258 arm_auxkc_init(void *mh, void *base)
 259 {
 260         /*
 261          * The kernel collection linker guarantees that the lowest vmaddr in an
 262          * AuxKC collection is 0 (but note that the mach header is higher up since
 263          * RW segments precede RO segments in the AuxKC).
 264          */
 265         uintptr_t slide = (uintptr_t)base;
 266         kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
 267
 268         assert(akc_mh->filetype == MH_FILESET);
 269         PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
 270
 271         /* rebase and sign jops */
 272         const void *collection_base_pointers[KCNumKinds];
 273         memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
 274         kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
 275
 276         kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
 277             (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
 278             (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
 279             (uintptr_t *)&segHIGHESTNLEAuxKC);
 280 #if defined(HAS_APPLE_PAC)
 281         OSRuntimeSignStructorsInFileset(akc_mh);
 282 #endif /* defined(HAS_APPLE_PAC) */
 283 }
 284
 285 #if HAS_IC_INVAL_FILTERS
 286 static void
 287 configure_misc_apple_regs(void)
 288 {
 289         uint64_t actlr, __unused acfg, __unused ahcr;
 290
 291         actlr = get_aux_control();
 292
 293 #if HAS_IC_INVAL_FILTERS
 294         ahcr = __builtin_arm_rsr64(ARM64_REG_AHCR_EL2);
 295         ahcr |= AHCR_IC_IVAU_EnRegime;
 296         ahcr |= AHCR_IC_IVAU_EnVMID;
 297         ahcr |= AHCR_IC_IALLU_EnRegime;
 298         ahcr |= AHCR_IC_IALLU_EnVMID;
 299         __builtin_arm_wsr64(ARM64_REG_AHCR_EL2, ahcr);
 300 #endif /* HAS_IC_INVAL_FILTERS */
 301
 302
 303 #if HAS_IC_INVAL_FILTERS
 304         actlr |= ACTLR_EL1_IC_IVAU_EnASID;
 305 #endif /* HAS_IC_INVAL_FILTERS */
 306
 307         set_aux_control(actlr);
 308
 309 }
 310 #endif /* HAS_IC_INVAL_FILTERS */
 311
 312 /*
 313  *              Routine:                arm_init
 314  *              Function:               Runs on the boot CPU, once, on entry from iBoot.
 315  */
 316
 317 __startup_func
 318 void
 319 arm_init(
 320         boot_args       *args)
 321 {
 322         unsigned int    maxmem;
 323         uint32_t        memsize;
 324         uint64_t        xmaxmem;
 325         thread_t        thread;
 326
 327         arm_slide_rebase_and_sign_image();
 328
 329         /* If kernel integrity is supported, use a constant copy of the boot args. */
 330         const_boot_args = *args;
 331         BootArgs = args = &const_boot_args;
 332
 333         cpu_data_init(&BootCpuData);
 334 #if defined(HAS_APPLE_PAC)
 335         /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
 336         BootCpuData.rop_key = KERNEL_ROP_ID;
 337         BootCpuData.jop_key = ml_default_jop_pid();
 338 #endif /* defined(HAS_APPLE_PAC) */
 339
 340         PE_init_platform(FALSE, args); /* Get platform expert set up */
 341
 342 #if __arm64__
 343         wfe_timeout_configure();
 344 #if HAS_IC_INVAL_FILTERS
 345         configure_misc_apple_regs();
 346 #endif /* HAS_IC_INVAL_FILTERS */
 347
 348 #if defined(HAS_APPLE_PAC)
 349 #if DEVELOPMENT || DEBUG
 350         boolean_t user_jop = TRUE;
 351         PE_parse_boot_argn("user_jop", &user_jop, sizeof(user_jop));
 352         if (!user_jop) {
 353                 args->bootFlags |= kBootFlagsDisableUserJOP;
 354         }
 355 #endif /* DEVELOPMENT || DEBUG */
 356         boolean_t user_ts_jop = TRUE;
 357         PE_parse_boot_argn("user_ts_jop", &user_ts_jop, sizeof(user_ts_jop));
 358         if (!user_ts_jop) {
 359                 args->bootFlags |= kBootFlagsDisableUserThreadStateJOP;
 360         }
 361         PE_parse_boot_argn("diversify_user_jop", &diversify_user_jop, sizeof(diversify_user_jop));
 362 #endif /* defined(HAS_APPLE_PAC) */
 363
 364         {
 365                 /*
 366                  * Select the advertised kernel page size.
 367                  */
 368                 if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
 369                         /*
 370                          * arm64 device with > 1GB of RAM:
 371                          * kernel uses 16KB pages.
 372                          */
 373                         PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 374                 } else {
 375                         /*
 376                          * arm64 device with <= 1GB of RAM:
 377                          * kernel uses hardware page size
 378                          * (4KB for H6/H7, 16KB for H8+).
 379                          */
 380                         PAGE_SHIFT_CONST = ARM_PGSHIFT;
 381                 }
 382
 383                 /* 32-bit apps always see 16KB page size */
 384                 page_shift_user32 = PAGE_MAX_SHIFT;
 385 #ifdef  APPLETYPHOON
 386                 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
 387                         if ((cpus_defeatures & 0xF) != 0) {
 388                                 cpu_defeatures_set(cpus_defeatures & 0xF);
 389                         }
 390                 }
 391 #endif
 392         }
 393 #endif
 394
 395         ml_parse_cpu_topology();
 396
 397         master_cpu = ml_get_boot_cpu_number();
 398         assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
 399
 400         BootCpuData.cpu_number = (unsigned short)master_cpu;
 401 #if     __arm__
 402         BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 403 #endif
 404         BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
 405         BootCpuData.istackptr = BootCpuData.intstack_top;
 406 #if __arm64__
 407         BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
 408         BootCpuData.excepstackptr = BootCpuData.excepstack_top;
 409 #else
 410         BootCpuData.fiqstack_top = (vm_offset_t) &fiqstack_top;
 411         BootCpuData.fiqstackptr = BootCpuData.fiqstack_top;
 412 #endif
 413         BootCpuData.cpu_console_buf = (void *)NULL;
 414         CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
 415         CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
 416             + ((uintptr_t)&BootCpuData
 417             - (uintptr_t)(args->virtBase)));
 418
 419         thread = thread_bootstrap();
 420         thread->machine.CpuDatap = &BootCpuData;
 421         thread->machine.pcpu_data_base = (vm_offset_t)0;
 422         machine_set_current_thread(thread);
 423
 424         /*
 425          * Preemption is enabled for this thread so that it can lock mutexes without
 426          * tripping the preemption check. In reality scheduling is not enabled until
 427          * this thread completes, and there are no other threads to switch to, so
 428          * preemption level is not really meaningful for the bootstrap thread.
 429          */
 430         thread->machine.preemption_count = 0;
 431 #if     __arm__ && __ARM_USER_PROTECT__
 432         {
 433                 unsigned int ttbr0_val, ttbr1_val;
 434                 __asm__ volatile ("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
 435                 __asm__ volatile ("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
 436                 thread->machine.uptw_ttb = ttbr0_val;
 437                 thread->machine.kptw_ttb = ttbr1_val;
 438         }
 439 #endif
 440         processor_t boot_processor = PERCPU_GET_MASTER(processor);
 441         boot_processor->kernel_timer = &thread->system_timer;
 442         boot_processor->thread_timer = &thread->system_timer;
 443
 444         cpu_bootstrap();
 445
 446         rtclock_early_init();
 447
 448         kernel_debug_string_early("kernel_startup_bootstrap");
 449         kernel_startup_bootstrap();
 450
 451         /*
 452          * Initialize the timer callout world
 453          */
 454         timer_call_init();
 455
 456         cpu_init();
 457
 458         processor_bootstrap();
 459
 460         if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
 461                 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
 462         } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
 463                 xmaxmem = (uint64_t) memsize;
 464         } else {
 465                 xmaxmem = 0;
 466         }
 467
 468 #if INTERRUPT_MASKED_DEBUG
 469         int wdt_boot_arg = 0;
 470         /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */
 471         if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug,
 472             sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg,
 473             sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1)) || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
 474                 interrupt_masked_debug = 0;
 475         }
 476
 477         PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout));
 478 #endif
 479
 480 #if HAS_BP_RET
 481         PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
 482         set_bp_ret(); // Apply branch predictor retention settings to boot CPU
 483 #endif
 484
 485         PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
 486
 487 #if __ARM_PAN_AVAILABLE__
 488         __builtin_arm_wsr("pan", 1);
 489 #endif  /* __ARM_PAN_AVAILABLE__ */
 490
 491         arm_vm_init(xmaxmem, args);
 492
 493         if (debug_boot_arg) {
 494                 patch_low_glo();
 495         }
 496
 497 #if __arm64__ && WITH_CLASSIC_S2R
 498         sleep_token_buffer_init();
 499 #endif
 500
 501         PE_consistent_debug_inherit();
 502
 503         /*
 504          * rdar://54622819 Insufficient HSP purge window can cause incorrect translation when ASID and TTBR base address is changed at same time)
 505          * (original info on HSP purge window issues can be found in rdar://55577508)
 506          * We need a flag to check for this, so calculate and set it here. We'll use it in machine_switch_amx_context().
 507          */
 508 #if __arm64__
 509         need_wa_rdar_55577508 = cpuid_get_cpufamily() == CPUFAMILY_ARM_LIGHTNING_THUNDER;
 510 #endif
 511
 512         /* setup debugging output if one has been chosen */
 513         kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
 514         kprintf("kprintf initialized\n");
 515
 516         serialmode = 0;
 517         if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
 518                 /* Do we want a serial keyboard and/or console? */
 519                 kprintf("Serial mode specified: %08X\n", serialmode);
 520                 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
 521                 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
 522                         if (force_sync) {
 523                                 serialmode |= SERIALMODE_SYNCDRAIN;
 524                                 kprintf(
 525                                         "WARNING: Forcing uart driver to output synchronously."
 526                                         "printf()s/IOLogs will impact kernel performance.\n"
 527                                         "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
 528                         }
 529                 }
 530         }
 531         if (kern_feature_override(KF_SERIAL_OVRD)) {
 532                 serialmode = 0;
 533         }
 534
 535         if (serialmode & SERIALMODE_OUTPUT) {                 /* Start serial if requested */
 536                 (void)switch_to_serial_console(); /* Switch into serial mode */
 537                 disableConsoleOutput = FALSE;     /* Allow printfs to happen */
 538         }
 539         PE_create_console();
 540
 541         /* setup console output */
 542         PE_init_printf(FALSE);
 543
 544 #if __arm64__
 545 #if DEBUG
 546         dump_kva_space();
 547 #endif
 548 #endif
 549
 550         cpu_machine_idle_init(TRUE);
 551
 552 #if     (__ARM_ARCH__ == 7)
 553         if (arm_diag & 0x8000) {
 554                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 555         }
 556 #endif
 557
 558         PE_init_platform(TRUE, &BootCpuData);
 559
 560 #if __arm64__
 561         ml_map_cpu_pio();
 562 #endif
 563
 564         cpu_timebase_init(TRUE);
 565         PE_init_cpu();
 566         fiq_context_init(TRUE);
 567
 568
 569 #if HIBERNATION
 570         pal_hib_init();
 571 #endif /* HIBERNATION */
 572
 573         /*
 574          * gPhysBase/Size only represent kernel-managed memory. These globals represent
 575          * the actual DRAM base address and size as reported by iBoot through the
 576          * device tree.
 577          */
 578         DTEntry chosen;
 579         unsigned int dt_entry_size;
 580         unsigned long const *dram_base;
 581         unsigned long const *dram_size;
 582         if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
 583                 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
 584         }
 585
 586         if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
 587                 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
 588         }
 589
 590         if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
 591                 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
 592         }
 593
 594         gDramBase = *dram_base;
 595         gDramSize = *dram_size;
 596
 597         /*
 598          * Initialize the stack protector for all future calls
 599          * to C code. Since kernel_bootstrap() eventually
 600          * switches stack context without returning through this
 601          * function, we do not risk failing the check even though
 602          * we mutate the guard word during execution.
 603          */
 604         __stack_chk_guard = (unsigned long)early_random();
 605         /* Zero a byte of the protector to guard
 606          * against string vulnerabilities
 607          */
 608         __stack_chk_guard &= ~(0xFFULL << 8);
 609         machine_startup(args);
 610 }
 611
 612 /*
 613  * Routine:        arm_init_cpu
 614  * Function:
 615  *    Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
 616  */
 617
 618 void
 619 arm_init_cpu(
 620         cpu_data_t      *cpu_data_ptr)
 621 {
 622 #if __ARM_PAN_AVAILABLE__
 623         __builtin_arm_wsr("pan", 1);
 624 #endif
 625
 626 #if HAS_IC_INVAL_FILTERS
 627         configure_misc_apple_regs();
 628 #endif /* HAS_IC_INVAL_FILTERS */
 629
 630         cpu_data_ptr->cpu_flags &= ~SleepState;
 631 #if     defined(ARMA7)
 632         cpu_data_ptr->cpu_CLW_active = 1;
 633 #endif
 634
 635         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 636
 637 #if HIBERNATION
 638         if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate)) {
 639                 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
 640                 extern uint64_t wake_abstime;
 641                 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
 642
 643                 // since the hw clock stops ticking across hibernation, we need to apply an offset;
 644                 // iBoot computes this offset for us and passes it via the hibernation header
 645                 extern uint64_t hwclock_conttime_offset;
 646                 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
 647
 648                 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
 649                 // fix up its preemption count
 650                 unsigned int expected_preemption_count = (gEnforceQuiesceSafety ? 2 : 1);
 651                 if (cpu_data_ptr->cpu_active_thread->machine.preemption_count != expected_preemption_count) {
 652                         panic("unexpected preemption count %u on boot cpu thread (should be %u)\n",
 653                             cpu_data_ptr->cpu_active_thread->machine.preemption_count,
 654                             expected_preemption_count);
 655                 }
 656                 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
 657         }
 658 #endif /* HIBERNATION */
 659
 660 #if __arm64__
 661         wfe_timeout_init();
 662         pmap_clear_user_ttb();
 663         flush_mmu_tlb();
 664 #endif
 665
 666         cpu_machine_idle_init(FALSE);
 667
 668         cpu_init();
 669
 670 #if     (__ARM_ARCH__ == 7)
 671         if (arm_diag & 0x8000) {
 672                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 673         }
 674 #endif
 675 #ifdef  APPLETYPHOON
 676         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 677                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 678         }
 679 #endif
 680         /* Initialize the timebase before serial_init, as some serial
 681          * drivers use mach_absolute_time() to implement rate control
 682          */
 683         cpu_timebase_init(FALSE);
 684
 685         if (cpu_data_ptr == &BootCpuData) {
 686 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 687                 /*
 688                  * Prevent CPUs from going into deep sleep until all
 689                  * CPUs are ready to do so.
 690                  */
 691                 arm64_stall_sleep = TRUE;
 692 #endif
 693                 serial_init();
 694                 PE_init_platform(TRUE, NULL);
 695                 commpage_update_timebase();
 696         }
 697         PE_init_cpu();
 698
 699         fiq_context_init(TRUE);
 700         cpu_data_ptr->rtcPop = EndOfAllTime;
 701         timer_resync_deadlines();
 702
 703 #if DEVELOPMENT || DEBUG
 704         PE_arm_debug_enable_trace();
 705 #endif
 706
 707
 708         kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
 709
 710         if (cpu_data_ptr == &BootCpuData) {
 711                 if (kdebug_enable == 0) {
 712                         __kdebug_only uint64_t elapsed = kdebug_wake();
 713                         KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
 714                 }
 715
 716 #if CONFIG_TELEMETRY
 717                 bootprofile_wake_from_sleep();
 718 #endif /* CONFIG_TELEMETRY */
 719         }
 720 #if MONOTONIC && defined(__arm64__)
 721         mt_wake_per_core();
 722 #endif /* MONOTONIC && defined(__arm64__) */
 723
 724 #if defined(KERNEL_INTEGRITY_CTRR)
 725         if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
 726                 lck_spin_lock(&ctrr_cpu_start_lck);
 727                 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
 728                 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
 729                 lck_spin_unlock(&ctrr_cpu_start_lck);
 730         }
 731 #endif
 732
 733         slave_main(NULL);
 734 }
 735
 736 /*
 737  * Routine:             arm_init_idle_cpu
 738  * Function:    Resume from non-retention WFI.  Called from the reset vector.
 739  */
 740 void __attribute__((noreturn))
 741 arm_init_idle_cpu(
 742         cpu_data_t      *cpu_data_ptr)
 743 {
 744 #if __ARM_PAN_AVAILABLE__
 745         __builtin_arm_wsr("pan", 1);
 746 #endif
 747 #if     defined(ARMA7)
 748         cpu_data_ptr->cpu_CLW_active = 1;
 749 #endif
 750
 751         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 752
 753 #if __arm64__
 754         wfe_timeout_init();
 755         pmap_clear_user_ttb();
 756         flush_mmu_tlb();
 757         /* Enable asynchronous exceptions */
 758         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 759 #endif
 760
 761 #if     (__ARM_ARCH__ == 7)
 762         if (arm_diag & 0x8000) {
 763                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 764         }
 765 #endif
 766 #ifdef  APPLETYPHOON
 767         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 768                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 769         }
 770 #endif
 771
 772         fiq_context_init(FALSE);
 773
 774         cpu_idle_exit(TRUE);
 775 }