osfmk/arm/arm_init.c

   1 /*
   2  * Copyright (c) 2007-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31
  32 #include <debug.h>
  33 #include <mach_ldebug.h>
  34 #include <mach_kdp.h>
  35
  36 #include <kern/misc_protos.h>
  37 #include <kern/thread.h>
  38 #include <kern/timer_queue.h>
  39 #include <kern/processor.h>
  40 #include <kern/startup.h>
  41 #include <kern/debug.h>
  42 #include <prng/random.h>
  43 #include <machine/machine_routines.h>
  44 #include <machine/commpage.h>
  45 /* ARM64_TODO unify boot.h */
  46 #if __arm64__
  47 #include <pexpert/arm64/boot.h>
  48 #elif __arm__
  49 #include <pexpert/arm/boot.h>
  50 #else
  51 #error Unsupported arch
  52 #endif
  53 #include <pexpert/arm/consistent_debug.h>
  54 #include <pexpert/device_tree.h>
  55 #include <arm/proc_reg.h>
  56 #include <arm/pmap.h>
  57 #include <arm/caches_internal.h>
  58 #include <arm/cpu_internal.h>
  59 #include <arm/cpu_data_internal.h>
  60 #include <arm/misc_protos.h>
  61 #include <arm/machine_cpu.h>
  62 #include <arm/rtclock.h>
  63 #include <vm/vm_map.h>
  64
  65 #include <libkern/kernel_mach_header.h>
  66 #include <libkern/stack_protector.h>
  67 #include <libkern/section_keywords.h>
  68 #include <san/kasan.h>
  69
  70 #include <pexpert/pexpert.h>
  71
  72 #include <console/serial_protos.h>
  73
  74 #if CONFIG_TELEMETRY
  75 #include <kern/telemetry.h>
  76 #endif
  77 #if MONOTONIC
  78 #include <kern/monotonic.h>
  79 #endif /* MONOTONIC */
  80
  81 extern void     patch_low_glo(void);
  82 extern int      serial_init(void);
  83 extern void sleep_token_buffer_init(void);
  84
  85 extern vm_offset_t intstack_top;
  86 #if __arm64__
  87 extern vm_offset_t excepstack_top;
  88 #else
  89 extern vm_offset_t fiqstack_top;
  90 #endif
  91
  92 extern const char version[];
  93 extern const char version_variant[];
  94 extern int      disableConsoleOutput;
  95
  96 int             pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
  97 int             pc_trace_cnt = PC_TRACE_BUF_SIZE;
  98 int             debug_task;
  99
 100 boolean_t up_style_idle_exit = 0;
 101
 102
 103
 104 #if INTERRUPT_MASKED_DEBUG
 105 boolean_t interrupt_masked_debug = 1;
 106 uint64_t interrupt_masked_timeout = 0xd0000;
 107 #endif
 108
 109 boot_args const_boot_args __attribute__((section("__DATA, __const")));
 110 boot_args      *BootArgs __attribute__((section("__DATA, __const")));
 111
 112 unsigned int arm_diag;
 113 #ifdef  APPLETYPHOON
 114 static unsigned cpus_defeatures = 0x0;
 115 extern void cpu_defeatures_set(unsigned int);
 116 #endif
 117
 118 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 119 extern volatile boolean_t arm64_stall_sleep;
 120 #endif
 121
 122 extern boolean_t force_immediate_debug_halt;
 123
 124 /*
 125  * Forward definition
 126  */
 127 void arm_init(boot_args * args);
 128
 129 #if __arm64__
 130 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
 131 #endif /* __arm64__ */
 132
 133
 134 /*
 135  * JOP rebasing
 136  */
 137
 138
 139 // Note, the following should come from a header from dyld
 140 static void
 141 rebase_chain(uintptr_t chainStartAddress, uint64_t stepMultiplier, uintptr_t baseAddress __unused, uint64_t slide)
 142 {
 143         uint64_t delta = 0;
 144         uintptr_t address = chainStartAddress;
 145         do {
 146                 uint64_t value = *(uint64_t*)address;
 147
 148                 bool isAuthenticated = (value & (1ULL << 63)) != 0;
 149                 bool isRebase = (value & (1ULL << 62)) == 0;
 150                 if (isRebase) {
 151                         if (isAuthenticated) {
 152                                 // The new value for a rebase is the low 32-bits of the threaded value plus the slide.
 153                                 uint64_t newValue = (value & 0xFFFFFFFF) + slide;
 154                                 // Add in the offset from the mach_header
 155                                 newValue += baseAddress;
 156                                 *(uint64_t*)address = newValue;
 157                         } else {
 158                                 // Regular pointer which needs to fit in 51-bits of value.
 159                                 // C++ RTTI uses the top bit, so we'll allow the whole top-byte
 160                                 // and the bottom 43-bits to be fit in to 51-bits.
 161                                 uint64_t top8Bits = value & 0x0007F80000000000ULL;
 162                                 uint64_t bottom43Bits = value & 0x000007FFFFFFFFFFULL;
 163                                 uint64_t targetValue = (top8Bits << 13) | (((intptr_t)(bottom43Bits << 21) >> 21) & 0x00FFFFFFFFFFFFFF);
 164                                 targetValue = targetValue + slide;
 165                                 *(uint64_t*)address = targetValue;
 166                         }
 167                 }
 168
 169                 // The delta is bits [51..61]
 170                 // And bit 62 is to tell us if we are a rebase (0) or bind (1)
 171                 value &= ~(1ULL << 62);
 172                 delta = (value & 0x3FF8000000000000) >> 51;
 173                 address += delta * stepMultiplier;
 174         } while (delta != 0);
 175 }
 176
 177 // Note, the following method should come from a header from dyld
 178 static bool
 179 rebase_threaded_starts(uint32_t *threadArrayStart, uint32_t *threadArrayEnd,
 180     uintptr_t macho_header_addr, uintptr_t macho_header_vmaddr, size_t slide)
 181 {
 182         uint32_t threadStartsHeader = *threadArrayStart;
 183         uint64_t stepMultiplier = (threadStartsHeader & 1) == 1 ? 8 : 4;
 184         for (uint32_t* threadOffset = threadArrayStart + 1; threadOffset != threadArrayEnd; ++threadOffset) {
 185                 if (*threadOffset == 0xFFFFFFFF) {
 186                         break;
 187                 }
 188                 rebase_chain(macho_header_addr + *threadOffset, stepMultiplier, macho_header_vmaddr, slide);
 189         }
 190         return true;
 191 }
 192
 193 /*
 194  *              Routine:                arm_init
 195  *              Function:
 196  */
 197
 198 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
 199 extern uint32_t __thread_starts_sect_end[]   __asm("section$end$__TEXT$__thread_starts");
 200
 201 void
 202 arm_init(
 203         boot_args       *args)
 204 {
 205         unsigned int    maxmem;
 206         uint32_t        memsize;
 207         uint64_t        xmaxmem;
 208         thread_t        thread;
 209         processor_t     my_master_proc;
 210
 211         // rebase and sign jops
 212         if (&__thread_starts_sect_end[0] != &__thread_starts_sect_start[0]) {
 213                 uintptr_t mh    = (uintptr_t) &_mh_execute_header;
 214                 uintptr_t slide = mh - VM_KERNEL_LINK_ADDRESS;
 215                 rebase_threaded_starts( &__thread_starts_sect_start[0],
 216                     &__thread_starts_sect_end[0],
 217                     mh, mh - slide, slide);
 218         }
 219
 220         /* If kernel integrity is supported, use a constant copy of the boot args. */
 221         const_boot_args = *args;
 222         BootArgs = args = &const_boot_args;
 223
 224         cpu_data_init(&BootCpuData);
 225
 226         PE_init_platform(FALSE, args); /* Get platform expert set up */
 227
 228 #if __arm64__
 229
 230
 231         {
 232                 unsigned int    tmp_16k = 0;
 233
 234 #ifdef  XXXX
 235                 /*
 236                  * Select the advertised kernel page size; without the boot-arg
 237                  * we default to the hardware page size for the current platform.
 238                  */
 239                 if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k))) {
 240                         PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 241                 } else {
 242                         PAGE_SHIFT_CONST = ARM_PGSHIFT;
 243                 }
 244 #else
 245                 /*
 246                  * Select the advertised kernel page size; with the boot-arg
 247                  * use to the hardware page size for the current platform.
 248                  */
 249                 int radar_20804515 = 1; /* default: new mode */
 250                 PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515));
 251                 if (radar_20804515) {
 252                         if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
 253                                 /*
 254                                  * arm64 device with > 1GB of RAM:
 255                                  * kernel uses 16KB pages.
 256                                  */
 257                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 258                         } else {
 259                                 /*
 260                                  * arm64 device with <= 1GB of RAM:
 261                                  * kernel uses hardware page size
 262                                  * (4KB for H6/H7, 16KB for H8+).
 263                                  */
 264                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 265                         }
 266                         /* 32-bit apps always see 16KB page size */
 267                         page_shift_user32 = PAGE_MAX_SHIFT;
 268                 } else {
 269                         /* kernel page size: */
 270                         if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k))) {
 271                                 PAGE_SHIFT_CONST = ARM_PGSHIFT;
 272                         } else {
 273                                 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
 274                         }
 275                         /* old mode: 32-bit apps see same page size as kernel */
 276                         page_shift_user32 = PAGE_SHIFT_CONST;
 277                 }
 278 #endif
 279 #ifdef  APPLETYPHOON
 280                 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
 281                         if ((cpus_defeatures & 0xF) != 0) {
 282                                 cpu_defeatures_set(cpus_defeatures & 0xF);
 283                         }
 284                 }
 285 #endif
 286         }
 287 #endif
 288
 289         ml_parse_cpu_topology();
 290
 291         master_cpu = ml_get_boot_cpu_number();
 292         assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
 293
 294         BootCpuData.cpu_number = (unsigned short)master_cpu;
 295 #if     __arm__
 296         BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 297 #endif
 298         BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
 299         BootCpuData.istackptr = BootCpuData.intstack_top;
 300 #if __arm64__
 301         BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
 302         BootCpuData.excepstackptr = BootCpuData.excepstack_top;
 303 #else
 304         BootCpuData.fiqstack_top = (vm_offset_t) &fiqstack_top;
 305         BootCpuData.fiqstackptr = BootCpuData.fiqstack_top;
 306 #endif
 307         BootCpuData.cpu_processor = cpu_processor_alloc(TRUE);
 308         BootCpuData.cpu_console_buf = (void *)NULL;
 309         CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
 310         CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
 311             + ((uintptr_t)&BootCpuData
 312             - (uintptr_t)(args->virtBase)));
 313
 314         thread_bootstrap();
 315         thread = current_thread();
 316         /*
 317          * Preemption is enabled for this thread so that it can lock mutexes without
 318          * tripping the preemption check. In reality scheduling is not enabled until
 319          * this thread completes, and there are no other threads to switch to, so
 320          * preemption level is not really meaningful for the bootstrap thread.
 321          */
 322         thread->machine.preemption_count = 0;
 323         thread->machine.CpuDatap = &BootCpuData;
 324 #if     __arm__ && __ARM_USER_PROTECT__
 325         {
 326                 unsigned int ttbr0_val, ttbr1_val, ttbcr_val;
 327                 __asm__ volatile ("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
 328                 __asm__ volatile ("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
 329                 __asm__ volatile ("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val));
 330                 thread->machine.uptw_ttb = ttbr0_val;
 331                 thread->machine.kptw_ttb = ttbr1_val;
 332                 thread->machine.uptw_ttc = ttbcr_val;
 333         }
 334 #endif
 335         BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer;
 336         BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer;
 337
 338         cpu_bootstrap();
 339
 340         rtclock_early_init();
 341
 342         kernel_early_bootstrap();
 343
 344         cpu_init();
 345
 346         EntropyData.index_ptr = EntropyData.buffer;
 347
 348         processor_bootstrap();
 349         my_master_proc = master_processor;
 350
 351         (void)PE_parse_boot_argn("diag", &arm_diag, sizeof(arm_diag));
 352
 353         if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
 354                 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
 355         } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
 356                 xmaxmem = (uint64_t) memsize;
 357         } else {
 358                 xmaxmem = 0;
 359         }
 360
 361         if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) {
 362                 up_style_idle_exit = 1;
 363         }
 364 #if INTERRUPT_MASKED_DEBUG
 365         int wdt_boot_arg = 0;
 366         /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */
 367         if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug,
 368             sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg,
 369             sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1))) {
 370                 interrupt_masked_debug = 0;
 371         }
 372
 373         PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout));
 374 #endif
 375
 376
 377
 378         PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
 379
 380 #if __ARM_PAN_AVAILABLE__
 381         __builtin_arm_wsr("pan", 1);
 382 #endif  /* __ARM_PAN_AVAILABLE__ */
 383
 384         arm_vm_init(xmaxmem, args);
 385
 386         uint32_t debugmode;
 387         if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) &&
 388             debugmode) {
 389                 patch_low_glo();
 390         }
 391
 392         printf_init();
 393         panic_init();
 394 #if __arm64__
 395         /* Enable asynchronous exceptions */
 396         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 397 #endif
 398 #if __arm64__ && WITH_CLASSIC_S2R
 399         sleep_token_buffer_init();
 400 #endif
 401
 402         PE_consistent_debug_inherit();
 403
 404         /* setup debugging output if one has been chosen */
 405         PE_init_kprintf(FALSE);
 406
 407         kprintf("kprintf initialized\n");
 408
 409         serialmode = 0;                                                      /* Assume normal keyboard and console */
 410         if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial
 411                                                                               * keyboard and/or
 412                                                                               * console? */
 413                 kprintf("Serial mode specified: %08X\n", serialmode);
 414                 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
 415                 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
 416                         if (force_sync) {
 417                                 serialmode |= SERIALMODE_SYNCDRAIN;
 418                                 kprintf(
 419                                         "WARNING: Forcing uart driver to output synchronously."
 420                                         "printf()s/IOLogs will impact kernel performance.\n"
 421                                         "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
 422                         }
 423                 }
 424         }
 425         if (kern_feature_override(KF_SERIAL_OVRD)) {
 426                 serialmode = 0;
 427         }
 428
 429         if (serialmode & SERIALMODE_OUTPUT) {                 /* Start serial if requested */
 430                 (void)switch_to_serial_console(); /* Switch into serial mode */
 431                 disableConsoleOutput = FALSE;     /* Allow printfs to happen */
 432         }
 433         PE_create_console();
 434
 435         /* setup console output */
 436         PE_init_printf(FALSE);
 437
 438 #if __arm64__
 439 #if DEBUG
 440         dump_kva_space();
 441 #endif
 442 #endif
 443
 444         cpu_machine_idle_init(TRUE);
 445
 446 #if     (__ARM_ARCH__ == 7)
 447         if (arm_diag & 0x8000) {
 448                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 449         }
 450 #endif
 451
 452         PE_init_platform(TRUE, &BootCpuData);
 453         cpu_timebase_init(TRUE);
 454         fiq_context_bootstrap(TRUE);
 455
 456
 457         /*
 458          * Initialize the stack protector for all future calls
 459          * to C code. Since kernel_bootstrap() eventually
 460          * switches stack context without returning through this
 461          * function, we do not risk failing the check even though
 462          * we mutate the guard word during execution.
 463          */
 464         __stack_chk_guard = (unsigned long)early_random();
 465         /* Zero a byte of the protector to guard
 466          * against string vulnerabilities
 467          */
 468         __stack_chk_guard &= ~(0xFFULL << 8);
 469         machine_startup(args);
 470 }
 471
 472 /*
 473  * Routine:        arm_init_cpu
 474  * Function:
 475  *    Re-initialize CPU when coming out of reset
 476  */
 477
 478 void
 479 arm_init_cpu(
 480         cpu_data_t      *cpu_data_ptr)
 481 {
 482 #if __ARM_PAN_AVAILABLE__
 483         __builtin_arm_wsr("pan", 1);
 484 #endif
 485
 486         cpu_data_ptr->cpu_flags &= ~SleepState;
 487 #if     __ARM_SMP__ && defined(ARMA7)
 488         cpu_data_ptr->cpu_CLW_active = 1;
 489 #endif
 490
 491         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 492
 493 #if __arm64__
 494         pmap_clear_user_ttb();
 495         flush_mmu_tlb();
 496         /* Enable asynchronous exceptions */
 497         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 498 #endif
 499
 500         cpu_machine_idle_init(FALSE);
 501
 502         cpu_init();
 503
 504 #if     (__ARM_ARCH__ == 7)
 505         if (arm_diag & 0x8000) {
 506                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 507         }
 508 #endif
 509 #ifdef  APPLETYPHOON
 510         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 511                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 512         }
 513 #endif
 514         /* Initialize the timebase before serial_init, as some serial
 515          * drivers use mach_absolute_time() to implement rate control
 516          */
 517         cpu_timebase_init(FALSE);
 518
 519         if (cpu_data_ptr == &BootCpuData) {
 520 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
 521                 /*
 522                  * Prevent CPUs from going into deep sleep until all
 523                  * CPUs are ready to do so.
 524                  */
 525                 arm64_stall_sleep = TRUE;
 526 #endif
 527                 serial_init();
 528                 PE_init_platform(TRUE, NULL);
 529                 commpage_update_timebase();
 530         }
 531
 532         fiq_context_init(TRUE);
 533         cpu_data_ptr->rtcPop = EndOfAllTime;
 534         timer_resync_deadlines();
 535
 536 #if DEVELOPMENT || DEBUG
 537         PE_arm_debug_enable_trace();
 538 #endif
 539
 540         kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id);
 541
 542         if (cpu_data_ptr == &BootCpuData) {
 543 #if CONFIG_TELEMETRY
 544                 bootprofile_wake_from_sleep();
 545 #endif /* CONFIG_TELEMETRY */
 546         }
 547 #if MONOTONIC && defined(__arm64__)
 548         mt_wake_per_core();
 549 #endif /* MONOTONIC && defined(__arm64__) */
 550
 551
 552         slave_main(NULL);
 553 }
 554
 555 /*
 556  * Routine:        arm_init_idle_cpu
 557  * Function:
 558  */
 559 void __attribute__((noreturn))
 560 arm_init_idle_cpu(
 561         cpu_data_t      *cpu_data_ptr)
 562 {
 563 #if __ARM_PAN_AVAILABLE__
 564         __builtin_arm_wsr("pan", 1);
 565 #endif
 566 #if     __ARM_SMP__ && defined(ARMA7)
 567         cpu_data_ptr->cpu_CLW_active = 1;
 568 #endif
 569
 570         machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
 571
 572 #if __arm64__
 573         pmap_clear_user_ttb();
 574         flush_mmu_tlb();
 575         /* Enable asynchronous exceptions */
 576         __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
 577 #endif
 578
 579 #if     (__ARM_ARCH__ == 7)
 580         if (arm_diag & 0x8000) {
 581                 set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC);
 582         }
 583 #endif
 584 #ifdef  APPLETYPHOON
 585         if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
 586                 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
 587         }
 588 #endif
 589
 590         fiq_context_init(FALSE);
 591
 592         cpu_idle_exit(TRUE);
 593 }