osfmk/arm/arm_init.c

   1 /*
   2  * Copyright (c) 2007-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 #include <debug.h>
  33 #include <mach_ldebug.h>
  34 #include <mach_kdp.h>
  35
  36 #include <kern/misc_protos.h>
  37 #include <kern/thread.h>
  38 #include <kern/timer_queue.h>
  39 #include <kern/processor.h>
  40 #include <kern/startup.h>
  41 #include <kern/debug.h>
  42 #include <prng/random.h>
  43 #include <machine/machine_routines.h>
  44 #include <machine/commpage.h>
  45 /* ARM64_TODO unify boot.h */
  46 #if __arm64__
  47 #include <pexpert/arm64/boot.h>
  48 #elif __arm__
  49 #include <pexpert/arm/boot.h>
  50 #else
  51 #error Unsupported arch
  52 #endif
  53 #include <pexpert/arm/consistent_debug.h>
  54 #include <pexpert/device_tree.h>
  55 #include <arm/proc_reg.h>
  56 #include <arm/pmap.h>
  57 #include <arm/caches_internal.h>
  58 #include <arm/cpu_internal.h>
  59 #include <arm/cpu_data_internal.h>
  60 #include <arm/misc_protos.h>
  61 #include <arm/machine_cpu.h>
  62 #include <arm/rtclock.h>
  63 #include <vm/vm_map.h>
  64
  65 #include <libkern/kernel_mach_header.h>
  66 #include <libkern/stack_protector.h>
  67 #include <libkern/section_keywords.h>
  68 #include <san/kasan.h>
  69
  70 #include <pexpert/pexpert.h>
  71
  72 #include <console/serial_protos.h>
  73
  74 #if CONFIG_TELEMETRY
  75 #include <kern/telemetry.h>
  76 #endif
  77 #if MONOTONIC
  78 #include <kern/monotonic.h>
  79 #endif /* MONOTONIC */
  80
  81 extern void     patch_low_glo(void);
  82 extern int      serial_init(void);
  83 extern void sleep_token_buffer_init(void);
  84
  85 extern vm_offset_t intstack_top;
  86 #if __arm64__
  87 extern vm_offset_t excepstack_top;
  88 extern uint64_t events_per_sec;
  89 #else
  90 extern vm_offset_t fiqstack_top;
  91 #endif
  92
  93 extern const char version[];
  94 extern const char version_variant[];
  95 extern int      disableConsoleOutput;
  96
  97 int             pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
  98 int             pc_trace_cnt = PC_TRACE_BUF_SIZE;
  99 int             debug_task;
 100
 101 boolean_t up_style_idle_exit = 0;
 102
 103
 104
 105
 106 #if INTERRUPT_MASKED_DEBUG
 107 boolean_t interrupt_masked_debug = 1;
 108 uint64_t interrupt_masked_timeout = 0xd0000;
 109 #endif
 110
 111 boot_args const_boot_args __attribute__((section("__DATA, __const")));
 112 boot_args      *BootArgs __attribute__((section("__DATA, __const")));
 113
 114 unsigned int arm_diag;
 115 #ifdef  APPLETYPHOON
 116 static unsigned cpus_defeatures = 0x0;
 117 extern void cpu_defeatures_set(unsigned int);
 118 #endif
 119
 120 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 121 extern volatile boolean_t arm64_stall_sleep;
 122 #endif
 123
 124 extern boolean_t force_immediate_debug_halt;
 125
 126 /*
 127  * Forward definition
 128  */
 129 void arm_init(boot_args * args);
 130
 131 #if __arm64__
 132 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
 133 #endif /* __arm64__ */
 134
 135
 136 /*
 137  * JOP rebasing
 138  */
 139
 140 #if defined(HAS_APPLE_PAC)
 141 #include <ptrauth.h>
 142 #endif /* defined(HAS_APPLE_PAC) */
 143
 144 // Note, the following should come from a header from dyld
 145 static void
 146 rebase_chain(uintptr_t chainStartAddress, uint64_t stepMultiplier, uintptr_t baseAddress __unused, uint64_t slide)
 147 {
 148         uint64_t delta = 0;
 149         uintptr_t address = chainStartAddress;
 150         do {
 151                 uint64_t value = *(uint64_t*)address;
 152
 153 #if HAS_APPLE_PAC
 154                 uint16_t diversity = (uint16_t)(value >> 32);
 155                 bool hasAddressDiversity = (value & (1ULL << 48)) != 0;
 156                 ptrauth_key key = (ptrauth_key)((value >> 49) & 0x3);
 157 #endif
 158                 bool isAuthenticated = (value & (1ULL << 63)) != 0;
 159                 bool isRebase = (value & (1ULL << 62)) == 0;
 160                 if (isRebase) {
 161                         if (isAuthenticated) {
 162                                 // The new value for a rebase is the low 32-bits of the threaded value plus the slide.
 163                                 uint64_t newValue = (value & 0xFFFFFFFF) + slide;
 164                                 // Add in the offset from the mach_header
 165                                 newValue += baseAddress;
 166 #if HAS_APPLE_PAC
 167                                 // We have bits to merge in to the discriminator
 168                                 uintptr_t discriminator = diversity;
 169                                 if (hasAddressDiversity) {
 170                                         // First calculate a new discriminator using the address of where we are trying to store the value
 171                                         // Only blend if we have a discriminator
 172                                         if (discriminator) {
 173                                                 discriminator = __builtin_ptrauth_blend_discriminator((void*)address, discriminator);
 174                                         } else {
 175                                                 discriminator = address;
 176                                         }
 177                                 }
 178                                 switch (key) {
 179                                 case ptrauth_key_asia:
 180                                         newValue = (uintptr_t)__builtin_ptrauth_sign_unauthenticated((void*)newValue, ptrauth_key_asia, discriminator);
 181                                         break;
 182                                 case ptrauth_key_asib:
 183                                         newValue = (uintptr_t)__builtin_ptrauth_sign_unauthenticated((void*)newValue, ptrauth_key_asib, discriminator);
 184                                         break;
 185                                 case ptrauth_key_asda:
 186                                         newValue = (uintptr_t)__builtin_ptrauth_sign_unauthenticated((void*)newValue, ptrauth_key_asda, discriminator);
 187                                         break;
 188                                 case ptrauth_key_asdb:
 189                                         newValue = (uintptr_t)__builtin_ptrauth_sign_unauthenticated((void*)newValue, ptrauth_key_asdb, discriminator);
 190                                         break;
 191                                 }
 192 #endif
 193                                 *(uint64_t*)address = newValue;
 194                         } else {
 195                                 // Regular pointer which needs to fit in 51-bits of value.
 196                                 // C++ RTTI uses the top bit, so we'll allow the whole top-byte
 197                                 // and the bottom 43-bits to be fit in to 51-bits.
 198                                 uint64_t top8Bits = value & 0x0007F80000000000ULL;
 199                                 uint64_t bottom43Bits = value & 0x000007FFFFFFFFFFULL;
 200                                 uint64_t targetValue = (top8Bits << 13) | (((intptr_t)(bottom43Bits << 21) >> 21) & 0x00FFFFFFFFFFFFFF);
 201                                 targetValue = targetValue + slide;
 202                                 *(uint64_t*)address = targetValue;
 203                         }
 204                 }
 205
 206                 // The delta is bits [51..61]
 207                 // And bit 62 is to tell us if we are a rebase (0) or bind (1)
 208                 value &= ~(1ULL << 62);
 209                 delta = (value & 0x3FF8000000000000) >> 51;
 210                 address += delta * stepMultiplier;
 211         } while (delta != 0);
 212 }
 213
 214 // Note, the following method should come from a header from dyld
 215 static bool
 216 rebase_threaded_starts(uint32_t *threadArrayStart, uint32_t *threadArrayEnd,
 217     uintptr_t macho_header_addr, uintptr_t macho_header_vmaddr, size_t slide)
 218 {
 219         uint32_t threadStartsHeader = *threadArrayStart;
 220         uint64_t stepMultiplier = (threadStartsHeader & 1) == 1 ? 8 : 4;
 221         for (uint32_t* threadOffset = threadArrayStart + 1; threadOffset != threadArrayEnd; ++threadOffset) {
 222                 if (*threadOffset == 0xFFFFFFFF) {
 223                         break;
 224                 }
 225                 rebase_chain(macho_header_addr + *threadOffset, stepMultiplier, macho_header_vmaddr, slide);
 226         }
 227         return true;
 228 }
 229
 230
 231 /*
 232  *              Routine:                arm_init
 233  *              Function:
 234  */
 235
 236 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
 237 extern uint32_t __thread_starts_sect_end[]   __asm("section$end$__TEXT$__thread_starts");
 238
 239 void
 240 arm_init(
 241         boot_args       *args)
 242 {
 243         unsigned int    maxmem;
 244         uint32_t        memsize;
 245         uint64_t        xmaxmem;
 246         thread_t        thread;
 247         processor_t     my_master_proc;
 248
 249         // rebase and sign jops
 250         if (&__thread_starts_sect_end[0] != &__thread_starts_sect_start[0]) {
 251                 uintptr_t mh    = (uintptr_t) &_mh_execute_header;
 252                 uintptr_t slide = mh - VM_KERNEL_LINK_ADDRESS;
 253                 rebase_threaded_starts( &__thread_starts_sect_start[0],
 254                     &__thread_starts_sect_end[0],
 255                     mh, mh - slide, slide);
 256         }
 257
 258         /* If kernel integrity is supported, use a constant copy of the boot args. */
 259         const_boot_args = *args;
 260         BootArgs = args = &const_boot_args;
 261
 262         cpu_data_init(&BootCpuData);
 263 #if defined(HAS_APPLE_PAC)
 264         /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
 265         BootCpuData.rop_key = KERNEL_ROP_ID;
 266 #endif /* defined(HAS_APPLE_PAC) */
 267
 268         PE_init_platform(FALSE, args); /* Get platform expert set up */
 269
 270 #if __arm64__
 271
 272
 273 #if defined(HAS_APPLE_PAC)
 274         boolean_t user_jop = TRUE;
 275         PE_parse_boot_argn("user_jop", &user_jop, sizeof(user_jop));
 276         if (!user_jop) {
 277                 args->bootFlags |= kBootFlagsDisableUserJOP;
 278         }
 279         boolean_t user_ts_jop = TRUE;
 280         PE_parse_boot_argn("user_ts_jop", &user_ts_jop, sizeof(user_ts_jop));
 281         if (!user_ts_jop) {
 282                 args->bootFlags |= kBootFlagsDisableUserThreadStateJOP;
 283         }
 284 #endif /* defined(HAS_APPLE_PAC) */
 285
 286         {
 287                 unsigned int    tmp_16k = 0;
 288
 289 #ifdef  XXXX
 290                 /*
 291                  * Select the advertised kernel page size; without the boot-arg
 292                  * we default to the hardware page size for the current platform.
 293                  */
 294                 if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k))) {
 295                         PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 296                 } else {
 297                         PAGE_SHIFT_CONST = ARM_PGSHIFT;
 298                 }
 299 #else
 300                 /*
 301                  * Select the advertised kernel page size; with the boot-arg
 302                  * use to the hardware page size for the current platform.
 303                  */
 304                 int radar_20804515 = 1; /* default: new mode */
 305                 PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515));
 306                 if (radar_20804515) {
 307                         if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
 308                                 /*
 309                                  * arm64 device with > 1GB of RAM:
 310                                  * kernel uses 16KB pages.
 311                                  */
 312                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 313                         } else {
 314                                 /*
 315                                  * arm64 device with <= 1GB of RAM:
 316                                  * kernel uses hardware page size
 317                                  * (4KB for H6/H7, 16KB for H8+).
 318                                  */
 319                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 320                         }
 321                         /* 32-bit apps always see 16KB page size */
 322                         page_shift_user32 = PAGE_MAX_SHIFT;
 323                 } else {
 324                         /* kernel page size: */
 325                         if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k))) {
 326                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 327                         } else {
 328                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 329                         }
 330                         /* old mode: 32-bit apps see same page size as kernel */
 331                         page_shift_user32 = PAGE_SHIFT_CONST;
 332                 }
 333 #endif
 334 #ifdef  APPLETYPHOON
 335                 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
 336                         if ((cpus_defeatures & 0xF) != 0) {
 337                                 cpu_defeatures_set(cpus_defeatures & 0xF);
 338                         }
 339                 }
 340 #endif
 341         }
 342 #endif
 343
 344         ml_parse_cpu_topology();
 345
 346         master_cpu = ml_get_boot_cpu_number();
 347         assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
 348
 349         BootCpuData.cpu_number = (unsigned short)master_cpu;
 350 #if     __arm__
 351         BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 352 #endif
 353         BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
 354         BootCpuData.istackptr = BootCpuData.intstack_top;
 355 #if __arm64__
 356         BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
 357         BootCpuData.excepstackptr = BootCpuData.excepstack_top;
 358 #else
 359         BootCpuData.fiqstack_top = (vm_offset_t) &fiqstack_top;
 360         BootCpuData.fiqstackptr = BootCpuData.fiqstack_top;
 361 #endif
 362         BootCpuData.cpu_processor = cpu_processor_alloc(TRUE);
 363         BootCpuData.cpu_console_buf = (void *)NULL;
 364         CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
 365         CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
 366             + ((uintptr_t)&BootCpuData
 367             - (uintptr_t)(args->virtBase)));
 368
 369         thread_bootstrap();
 370         thread = current_thread();
 371         /*
 372          * Preemption is enabled for this thread so that it can lock mutexes without
 373          * tripping the preemption check. In reality scheduling is not enabled until
 374          * this thread completes, and there are no other threads to switch to, so
 375          * preemption level is not really meaningful for the bootstrap thread.
 376          */
 377         thread->machine.preemption_count = 0;
 378         thread->machine.CpuDatap = &BootCpuData;
 379 #if     __arm__ && __ARM_USER_PROTECT__
 380         {
 381                 unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
 382                 __asm__ volatile ("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
 383                 __asm__ volatile ("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
 384                 __asm__ volatile ("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
 385                 thread->machine.uptw_ttb = ttbr0_val;
 386                 thread->machine.kptw_ttb = ttbr1_val;
 387                 thread->machine.uptw_ttc = ttbcr_val;
 388         }
 389 #endif
 390         BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer;
 391         BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer;
 392
 393         cpu_bootstrap();
 394
 395         rtclock_early_init();
 396
 397         lck_mod_init();
 398
 399         /*
 400          * Initialize the timer callout world
 401          */
 402         timer_call_init();
 403
 404         kernel_early_bootstrap();
 405
 406         cpu_init();
 407
 408         processor_bootstrap();
 409         my_master_proc = master_processor;
 410
 411         (void)PE_parse_boot_argn("diag", &arm_diag, sizeof(arm_diag));
 412
 413         if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
 414                 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
 415         } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
 416                 xmaxmem = (uint64_t) memsize;
 417         } else {
 418                 xmaxmem = 0;
 419         }
 420
 421         if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) {
 422                 up_style_idle_exit = 1;
 423         }
 424 #if INTERRUPT_MASKED_DEBUG
 425         int wdt_boot_arg = 0;
 426         /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */
 427         if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug,
 428             sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg,
 429             sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1)) || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
 430                 interrupt_masked_debug = 0;
 431         }
 432
 433         PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout));
 434 #endif
 435
 436
 437
 438         PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
 439
 440 #if __ARM_PAN_AVAILABLE__
 441         __builtin_arm_wsr("pan", 1);
 442 #endif  /* __ARM_PAN_AVAILABLE__ */
 443
 444         arm_vm_init(xmaxmem, args);
 445
 446         uint32_t debugmode;
 447         if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) &&
 448             debugmode) {
 449                 patch_low_glo();
 450         }
 451
 452         printf_init();
 453         panic_init();
 454 #if __arm64__
 455         /* Enable asynchronous exceptions */
 456         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 457 #endif
 458 #if __arm64__ && WITH_CLASSIC_S2R
 459         sleep_token_buffer_init();
 460 #endif
 461
 462         PE_consistent_debug_inherit();
 463
 464         /* setup debugging output if one has been chosen */
 465         PE_init_kprintf(FALSE);
 466
 467         kprintf("kprintf initialized\n");
 468
 469         serialmode = 0;                                                      /* Assume normal keyboard and console */
 470         if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial
 471                                                                               * keyboard and/or
 472                                                                               * console? */
 473                 kprintf("Serial mode specified: %08X\n", serialmode);
 474                 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
 475                 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
 476                         if (force_sync) {
 477                                 serialmode |= SERIALMODE_SYNCDRAIN;
 478                                 kprintf(
 479                                         "WARNING: Forcing uart driver to output synchronously."
 480                                         "printf()s/IOLogs will impact kernel performance.\n"
 481                                         "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
 482                         }
 483                 }
 484         }
 485         if (kern_feature_override(KF_SERIAL_OVRD)) {
 486                 serialmode = 0;
 487         }
 488
 489         if (serialmode & SERIALMODE_OUTPUT) {                 /* Start serial if requested */
 490                 (void)switch_to_serial_console(); /* Switch into serial mode */
 491                 disableConsoleOutput = FALSE;     /* Allow printfs to happen */
 492         }
 493         PE_create_console();
 494
 495         /* setup console output */
 496         PE_init_printf(FALSE);
 497
 498 #if __arm64__
 499 #if DEBUG
 500         dump_kva_space();
 501 #endif
 502 #endif
 503
 504         cpu_machine_idle_init(TRUE);
 505
 506 #if     (__ARM_ARCH__ == 7)
 507         if (arm_diag & 0x8000) {
 508                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 509         }
 510 #endif
 511
 512         PE_init_platform(TRUE, &BootCpuData);
 513
 514 #if __arm64__
 515         if (PE_parse_boot_argn("wfe_events_sec", &events_per_sec, sizeof(events_per_sec))) {
 516                 if (events_per_sec <= 0) {
 517                         events_per_sec = 1;
 518                 } else if (events_per_sec > USEC_PER_SEC) {
 519                         events_per_sec = USEC_PER_SEC;
 520                 }
 521         } else {
 522 #if defined(ARM_BOARD_WFE_TIMEOUT_NS)
 523                 events_per_sec = NSEC_PER_SEC / ARM_BOARD_WFE_TIMEOUT_NS;
 524 #else /* !defined(ARM_BOARD_WFE_TIMEOUT_NS) */
 525                 /* Default to 1usec (or as close as we can get) */
 526                 events_per_sec = USEC_PER_SEC;
 527 #endif /* !defined(ARM_BOARD_WFE_TIMEOUT_NS) */
 528         }
 529 #endif
 530
 531         cpu_timebase_init(TRUE);
 532         PE_init_cpu();
 533         fiq_context_bootstrap(TRUE);
 534
 535
 536         /*
 537          * Initialize the stack protector for all future calls
 538          * to C code. Since kernel_bootstrap() eventually
 539          * switches stack context without returning through this
 540          * function, we do not risk failing the check even though
 541          * we mutate the guard word during execution.
 542          */
 543         __stack_chk_guard = (unsigned long)early_random();
 544         /* Zero a byte of the protector to guard
 545          * against string vulnerabilities
 546          */
 547         __stack_chk_guard &= ~(0xFFULL << 8);
 548         machine_startup(args);
 549 }
 550
 551 /*
 552  * Routine:        arm_init_cpu
 553  * Function:
 554  *    Re-initialize CPU when coming out of reset
 555  */
 556
 557 void
 558 arm_init_cpu(
 559         cpu_data_t      *cpu_data_ptr)
 560 {
 561 #if __ARM_PAN_AVAILABLE__
 562         __builtin_arm_wsr("pan", 1);
 563 #endif
 564
 565
 566         cpu_data_ptr->cpu_flags &= ~SleepState;
 567 #if     __ARM_SMP__ && defined(ARMA7)
 568         cpu_data_ptr->cpu_CLW_active = 1;
 569 #endif
 570
 571         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 572
 573 #if __arm64__
 574         pmap_clear_user_ttb();
 575         flush_mmu_tlb();
 576         /* Enable asynchronous exceptions */
 577         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 578 #endif
 579
 580         cpu_machine_idle_init(FALSE);
 581
 582         cpu_init();
 583
 584 #if     (__ARM_ARCH__ == 7)
 585         if (arm_diag & 0x8000) {
 586                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 587         }
 588 #endif
 589 #ifdef  APPLETYPHOON
 590         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 591                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 592         }
 593 #endif
 594         /* Initialize the timebase before serial_init, as some serial
 595          * drivers use mach_absolute_time() to implement rate control
 596          */
 597         cpu_timebase_init(FALSE);
 598
 599         if (cpu_data_ptr == &BootCpuData) {
 600 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 601                 /*
 602                  * Prevent CPUs from going into deep sleep until all
 603                  * CPUs are ready to do so.
 604                  */
 605                 arm64_stall_sleep = TRUE;
 606 #endif
 607                 serial_init();
 608                 PE_init_platform(TRUE, NULL);
 609                 commpage_update_timebase();
 610         }
 611         PE_init_cpu();
 612
 613         fiq_context_init(TRUE);
 614         cpu_data_ptr->rtcPop = EndOfAllTime;
 615         timer_resync_deadlines();
 616
 617 #if DEVELOPMENT || DEBUG
 618         PE_arm_debug_enable_trace();
 619 #endif
 620
 621         kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id);
 622
 623         if (cpu_data_ptr == &BootCpuData) {
 624 #if CONFIG_TELEMETRY
 625                 bootprofile_wake_from_sleep();
 626 #endif /* CONFIG_TELEMETRY */
 627         }
 628 #if MONOTONIC && defined(__arm64__)
 629         mt_wake_per_core();
 630 #endif /* MONOTONIC && defined(__arm64__) */
 631
 632
 633         slave_main(NULL);
 634 }
 635
 636 /*
 637  * Routine:        arm_init_idle_cpu
 638  * Function:
 639  */
 640 void __attribute__((noreturn))
 641 arm_init_idle_cpu(
 642         cpu_data_t      *cpu_data_ptr)
 643 {
 644 #if __ARM_PAN_AVAILABLE__
 645         __builtin_arm_wsr("pan", 1);
 646 #endif
 647 #if     __ARM_SMP__ && defined(ARMA7)
 648         cpu_data_ptr->cpu_CLW_active = 1;
 649 #endif
 650
 651         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 652
 653 #if __arm64__
 654         pmap_clear_user_ttb();
 655         flush_mmu_tlb();
 656         /* Enable asynchronous exceptions */
 657         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 658 #endif
 659
 660 #if     (__ARM_ARCH__ == 7)
 661         if (arm_diag & 0x8000) {
 662                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 663         }
 664 #endif
 665 #ifdef  APPLETYPHOON
 666         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 667                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 668         }
 669 #endif
 670
 671         fiq_context_init(FALSE);
 672
 673         cpu_idle_exit(TRUE);
 674 }