osfmk/arm/cpu.c

   1 /*
   2  * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  *      File:   arm/cpu.c
  30  *
  31  *      cpu specific routines
  32  */
  33
  34 #include <kern/kalloc.h>
  35 #include <kern/machine.h>
  36 #include <kern/cpu_number.h>
  37 #include <kern/percpu.h>
  38 #include <kern/thread.h>
  39 #include <kern/timer_queue.h>
  40 #include <arm/cpu_data.h>
  41 #include <arm/cpuid.h>
  42 #include <arm/caches_internal.h>
  43 #include <arm/cpu_data_internal.h>
  44 #include <arm/cpu_internal.h>
  45 #include <arm/misc_protos.h>
  46 #include <arm/machine_cpu.h>
  47 #include <arm/rtclock.h>
  48 #include <arm/proc_reg.h>
  49 #include <mach/processor_info.h>
  50 #include <vm/pmap.h>
  51 #include <vm/vm_kern.h>
  52 #include <vm/vm_map.h>
  53 #include <pexpert/arm/board_config.h>
  54 #include <pexpert/arm/protos.h>
  55 #include <sys/kdebug.h>
  56
  57 #include <machine/atomic.h>
  58
  59 #if KPC
  60 #include <kern/kpc.h>
  61 #endif
  62
  63 extern unsigned int resume_idle_cpu;
  64 extern unsigned int start_cpu;
  65
  66 unsigned int   start_cpu_paddr;
  67
  68 extern boolean_t        idle_enable;
  69 extern unsigned int     real_ncpus;
  70 extern uint64_t         wake_abstime;
  71
  72 extern void* wfi_inst;
  73 unsigned wfi_fast = 1;
  74 unsigned patch_to_nop = 0xe1a00000;
  75
  76 void    *LowExceptionVectorsAddr;
  77 #define IOS_STATE               (((vm_offset_t)LowExceptionVectorsAddr + 0x80))
  78 #define IOS_STATE_SIZE  (0x08UL)
  79 static const uint8_t suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
  80 static const uint8_t running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
  81
  82 /*
  83  *      Routine:        cpu_bootstrap
  84  *      Function:
  85  */
  86 void
  87 cpu_bootstrap(void)
  88 {
  89 }
  90
  91
  92 /*
  93  *      Routine:        cpu_sleep
  94  *      Function:
  95  */
  96 void
  97 cpu_sleep(void)
  98 {
  99         cpu_data_t     *cpu_data_ptr = getCpuDatap();
 100         pmap_switch_user_ttb(kernel_pmap);
 101         cpu_data_ptr->cpu_active_thread = current_thread();
 102         cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
 103         cpu_data_ptr->cpu_flags |= SleepState;
 104         cpu_data_ptr->cpu_user_debug = NULL;
 105
 106         CleanPoC_Dcache();
 107
 108         PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
 109 }
 110
 111 _Atomic uint32_t cpu_idle_count = 0;
 112
 113 /*
 114  *      Routine:        cpu_idle
 115  *      Function:
 116  */
 117 void __attribute__((noreturn))
 118 cpu_idle(void)
 119 {
 120         cpu_data_t     *cpu_data_ptr = getCpuDatap();
 121         uint64_t        new_idle_timeout_ticks = 0x0ULL, lastPop;
 122
 123         if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) {
 124                 Idle_load_context();
 125         }
 126         if (!SetIdlePop()) {
 127                 Idle_load_context();
 128         }
 129         lastPop = cpu_data_ptr->rtcPop;
 130
 131         pmap_switch_user_ttb(kernel_pmap);
 132         cpu_data_ptr->cpu_active_thread = current_thread();
 133         if (cpu_data_ptr->cpu_user_debug) {
 134                 arm_debug_set(NULL);
 135         }
 136         cpu_data_ptr->cpu_user_debug = NULL;
 137
 138         if (cpu_data_ptr->cpu_idle_notify != NULL) {
 139                 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
 140         }
 141
 142         if (cpu_data_ptr->idle_timer_notify != 0) {
 143                 if (new_idle_timeout_ticks == 0x0ULL) {
 144                         /* turn off the idle timer */
 145                         cpu_data_ptr->idle_timer_deadline = 0x0ULL;
 146                 } else {
 147                         /* set the new idle timeout */
 148                         clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
 149                 }
 150                 timer_resync_deadlines();
 151                 if (cpu_data_ptr->rtcPop != lastPop) {
 152                         /*
 153                          * Ignore the return value here: this CPU has called idle_notify and
 154                          * committed to going idle.
 155                          */
 156                         SetIdlePop();
 157                 }
 158         }
 159
 160 #if KPC
 161         kpc_idle();
 162 #endif /* KPC */
 163
 164         platform_cache_idle_enter();
 165         cpu_idle_wfi((boolean_t) wfi_fast);
 166         platform_cache_idle_exit();
 167
 168         ClearIdlePop(TRUE);
 169         cpu_idle_exit(FALSE);
 170 }
 171
 172 /*
 173  *      Routine:        cpu_idle_exit
 174  *      Function:
 175  */
 176 void
 177 cpu_idle_exit(boolean_t from_reset __unused)
 178 {
 179         uint64_t        new_idle_timeout_ticks = 0x0ULL;
 180         cpu_data_t     *cpu_data_ptr = getCpuDatap();
 181
 182 #if KPC
 183         kpc_idle_exit();
 184 #endif
 185
 186
 187         pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread());
 188
 189         if (cpu_data_ptr->cpu_idle_notify != NULL) {
 190                 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
 191         }
 192
 193         if (cpu_data_ptr->idle_timer_notify != 0) {
 194                 if (new_idle_timeout_ticks == 0x0ULL) {
 195                         /* turn off the idle timer */
 196                         cpu_data_ptr->idle_timer_deadline = 0x0ULL;
 197                 } else {
 198                         /* set the new idle timeout */
 199                         clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
 200                 }
 201                 timer_resync_deadlines();
 202         }
 203
 204         Idle_load_context();
 205 }
 206
 207 void
 208 cpu_init(void)
 209 {
 210         cpu_data_t     *cdp = getCpuDatap();
 211         arm_cpu_info_t *cpu_info_p;
 212
 213         if (cdp->cpu_type != CPU_TYPE_ARM) {
 214                 cdp->cpu_type = CPU_TYPE_ARM;
 215
 216                 timer_call_queue_init(&cdp->rtclock_timer.queue);
 217                 cdp->rtclock_timer.deadline = EndOfAllTime;
 218
 219                 if (cdp == &BootCpuData) {
 220                         do_cpuid();
 221                         do_cacheid();
 222                         do_mvfpid();
 223                 } else {
 224                         /*
 225                          * We initialize non-boot CPUs here; the boot CPU is
 226                          * dealt with as part of pmap_bootstrap.
 227                          */
 228                         pmap_cpu_data_init();
 229                 }
 230                 /* ARM_SMP: Assuming identical cpu */
 231                 do_debugid();
 232
 233                 cpu_info_p = cpuid_info();
 234
 235                 /* switch based on CPU's reported architecture */
 236                 switch (cpu_info_p->arm_info.arm_arch) {
 237                 case CPU_ARCH_ARMv4T:
 238                 case CPU_ARCH_ARMv5T:
 239                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V4T;
 240                         break;
 241                 case CPU_ARCH_ARMv5TE:
 242                 case CPU_ARCH_ARMv5TEJ:
 243                         if (cpu_info_p->arm_info.arm_implementor == CPU_VID_INTEL) {
 244                                 cdp->cpu_subtype = CPU_SUBTYPE_ARM_XSCALE;
 245                         } else {
 246                                 cdp->cpu_subtype = CPU_SUBTYPE_ARM_V5TEJ;
 247                         }
 248                         break;
 249                 case CPU_ARCH_ARMv6:
 250                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V6;
 251                         break;
 252                 case CPU_ARCH_ARMv7:
 253                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7;
 254                         break;
 255                 case CPU_ARCH_ARMv7f:
 256                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7F;
 257                         break;
 258                 case CPU_ARCH_ARMv7s:
 259                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7S;
 260                         break;
 261                 case CPU_ARCH_ARMv7k:
 262                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7K;
 263                         break;
 264                 default:
 265                         cdp->cpu_subtype = CPU_SUBTYPE_ARM_ALL;
 266                         break;
 267                 }
 268
 269                 cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
 270         }
 271         cdp->cpu_stat.irq_ex_cnt_wake = 0;
 272         cdp->cpu_stat.ipi_cnt_wake = 0;
 273         cdp->cpu_running = TRUE;
 274         cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
 275         cdp->cpu_sleep_token = 0x0UL;
 276 }
 277
 278 void
 279 cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
 280 {
 281         vm_offset_t             irq_stack = 0;
 282         vm_offset_t             fiq_stack = 0;
 283
 284         kern_return_t kr = kernel_memory_allocate(kernel_map, &irq_stack,
 285             INTSTACK_SIZE + (2 * PAGE_SIZE),
 286             PAGE_MASK,
 287             KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
 288             VM_KERN_MEMORY_STACK);
 289         if (kr != KERN_SUCCESS) {
 290                 panic("Unable to allocate cpu interrupt stack\n");
 291         }
 292
 293         cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
 294         cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;
 295
 296         kr = kernel_memory_allocate(kernel_map, &fiq_stack,
 297             FIQSTACK_SIZE + (2 * PAGE_SIZE),
 298             PAGE_MASK,
 299             KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
 300             VM_KERN_MEMORY_STACK);
 301         if (kr != KERN_SUCCESS) {
 302                 panic("Unable to allocate cpu exception stack\n");
 303         }
 304
 305         cpu_data_ptr->fiqstack_top = fiq_stack + PAGE_SIZE + FIQSTACK_SIZE;
 306         cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top;
 307 }
 308
 309 void
 310 cpu_data_free(cpu_data_t *cpu_data_ptr)
 311 {
 312         if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
 313                 return;
 314         }
 315
 316         int cpu_number = cpu_data_ptr->cpu_number;
 317
 318         if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
 319                 OSDecrementAtomic((SInt32*)&real_ncpus);
 320                 CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
 321                 CpuDataEntries[cpu_number].cpu_data_paddr = 0;
 322                 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
 323         }
 324         (kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
 325         (kfree)((void *)(cpu_data_ptr->fiqstack_top - FIQSTACK_SIZE), FIQSTACK_SIZE);
 326 }
 327
 328 void
 329 cpu_data_init(cpu_data_t *cpu_data_ptr)
 330 {
 331         cpu_data_ptr->cpu_flags = 0;
 332 #if     __arm__
 333         cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 334 #endif
 335         cpu_data_ptr->cpu_int_state = 0;
 336         cpu_data_ptr->cpu_pending_ast = AST_NONE;
 337         cpu_data_ptr->cpu_cache_dispatch = NULL;
 338         cpu_data_ptr->rtcPop = EndOfAllTime;
 339         cpu_data_ptr->rtclock_datap = &RTClockData;
 340         cpu_data_ptr->cpu_user_debug = NULL;
 341         cpu_data_ptr->cpu_base_timebase_low = 0;
 342         cpu_data_ptr->cpu_base_timebase_high = 0;
 343         cpu_data_ptr->cpu_idle_notify = NULL;
 344         cpu_data_ptr->cpu_idle_latency = 0x0ULL;
 345         cpu_data_ptr->cpu_idle_pop = 0x0ULL;
 346         cpu_data_ptr->cpu_reset_type = 0x0UL;
 347         cpu_data_ptr->cpu_reset_handler = 0x0UL;
 348         cpu_data_ptr->cpu_reset_assist = 0x0UL;
 349         cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
 350         cpu_data_ptr->cpu_phys_id = 0x0UL;
 351         cpu_data_ptr->cpu_l2_access_penalty = 0;
 352         cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
 353         cpu_data_ptr->cpu_cluster_id = 0;
 354         cpu_data_ptr->cpu_l2_id = 0;
 355         cpu_data_ptr->cpu_l2_size = 0;
 356         cpu_data_ptr->cpu_l3_id = 0;
 357         cpu_data_ptr->cpu_l3_size = 0;
 358
 359         cpu_data_ptr->cpu_signal = SIGPdisabled;
 360
 361         cpu_data_ptr->cpu_get_fiq_handler = NULL;
 362         cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
 363         cpu_data_ptr->cpu_tbd_hardware_val = NULL;
 364         cpu_data_ptr->cpu_get_decrementer_func = NULL;
 365         cpu_data_ptr->cpu_set_decrementer_func = NULL;
 366         cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
 367         cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
 368         cpu_data_ptr->cpu_xcall_p0 = NULL;
 369         cpu_data_ptr->cpu_xcall_p1 = NULL;
 370         cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
 371         cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
 372
 373 #if     defined(ARMA7)
 374         cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL;
 375         cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL;
 376         cpu_data_ptr->cpu_CLWClean_req = 0x0ULL;
 377         cpu_data_ptr->cpu_CLWClean_last = 0x0ULL;
 378         cpu_data_ptr->cpu_CLW_active = 0x1UL;
 379 #endif
 380
 381 #if !XNU_MONITOR
 382         pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
 383
 384         pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL;
 385         pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0;
 386         pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
 387
 388         bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
 389 #endif
 390         cpu_data_ptr->halt_status = CPU_NOT_HALTED;
 391 }
 392
 393 kern_return_t
 394 cpu_data_register(cpu_data_t *cpu_data_ptr)
 395 {
 396         int cpu;
 397
 398         cpu = OSIncrementAtomic((SInt32*)&real_ncpus);
 399         if (real_ncpus > ml_get_cpu_count()) {
 400                 return KERN_FAILURE;
 401         }
 402
 403         cpu_data_ptr->cpu_number = cpu;
 404         __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
 405         CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
 406         CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
 407         return KERN_SUCCESS;
 408 }
 409
 410 kern_return_t
 411 cpu_start(int cpu)
 412 {
 413         kprintf("cpu_start() cpu: %d\n", cpu);
 414         if (cpu == cpu_number()) {
 415                 cpu_machine_init();
 416                 return KERN_SUCCESS;
 417         } else {
 418                 cpu_data_t      *cpu_data_ptr;
 419                 thread_t        first_thread;
 420                 processor_t     processor;
 421
 422                 cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
 423                 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
 424
 425 #if !XNU_MONITOR
 426                 cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL;
 427 #endif
 428
 429                 processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
 430                 if (processor->startup_thread != THREAD_NULL) {
 431                         first_thread = processor->startup_thread;
 432                 } else {
 433                         first_thread = processor->idle_thread;
 434                 }
 435                 cpu_data_ptr->cpu_active_thread = first_thread;
 436                 first_thread->machine.CpuDatap = cpu_data_ptr;
 437                 first_thread->machine.pcpu_data_base =
 438                     (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
 439
 440                 flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
 441                 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
 442                 (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
 443                 return KERN_SUCCESS;
 444         }
 445 }
 446
 447 void
 448 cpu_timebase_init(boolean_t from_boot __unused)
 449 {
 450         cpu_data_t *cdp = getCpuDatap();
 451
 452         if (cdp->cpu_get_fiq_handler == NULL) {
 453                 cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
 454                 cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
 455                 cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
 456                 cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
 457                 cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
 458         }
 459         cdp->cpu_decrementer = 0x7FFFFFFFUL;
 460         cdp->cpu_timebase_low = 0x0UL;
 461         cdp->cpu_timebase_high = 0x0UL;
 462
 463 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
 464         /* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
 465          * are 32-bit. */
 466         cdp->cpu_base_timebase_low = rtclock_base_abstime_low;
 467         cdp->cpu_base_timebase_high = rtclock_base_abstime_high;
 468 #else
 469         *((uint64_t *) &cdp->cpu_base_timebase_low) = rtclock_base_abstime;
 470 #endif
 471 }
 472
 473
 474 __attribute__((noreturn))
 475 void
 476 ml_arm_sleep(void)
 477 {
 478         cpu_data_t     *cpu_data_ptr = getCpuDatap();
 479
 480         if (cpu_data_ptr == &BootCpuData) {
 481                 cpu_data_t      *target_cdp;
 482                 unsigned int    cpu;
 483
 484                 const unsigned int max_cpu_id = ml_get_max_cpu_number();
 485                 for (cpu = 0; cpu <= max_cpu_id; cpu++) {
 486                         target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
 487                         if (target_cdp == (cpu_data_t *)NULL) {
 488                                 break;
 489                         }
 490
 491                         if (target_cdp == cpu_data_ptr) {
 492                                 continue;
 493                         }
 494
 495                         while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
 496                                 ;
 497                         }
 498                 }
 499
 500                 /* Now that the other cores have entered the sleep path, set
 501                  * the abstime fixup we'll use when we resume.*/
 502                 rtclock_base_abstime = ml_get_timebase();
 503                 wake_abstime = rtclock_base_abstime;
 504         } else {
 505                 platform_cache_disable();
 506                 CleanPoU_Dcache();
 507         }
 508         cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
 509 #if     defined(ARMA7)
 510         cpu_data_ptr->cpu_CLWFlush_req = 0;
 511         cpu_data_ptr->cpu_CLWClean_req = 0;
 512         __builtin_arm_dmb(DMB_ISH);
 513         cpu_data_ptr->cpu_CLW_active = 0;
 514 #endif
 515         if (cpu_data_ptr == &BootCpuData) {
 516                 platform_cache_disable();
 517                 platform_cache_shutdown();
 518                 bcopy((const void *)suspend_signature, (void *)(IOS_STATE), IOS_STATE_SIZE);
 519         } else {
 520                 CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
 521         }
 522
 523         __builtin_arm_dsb(DSB_SY);
 524         while (TRUE) {
 525 #if     __ARM_ENABLE_WFE_
 526                 __builtin_arm_wfe();
 527 #endif
 528         } /* Spin */
 529 }
 530
 531 void
 532 cpu_machine_idle_init(boolean_t from_boot)
 533 {
 534         static const unsigned int       *BootArgs_paddr = (unsigned int *)NULL;
 535         static const unsigned int       *CpuDataEntries_paddr = (unsigned int *)NULL;
 536         static unsigned int             resume_idle_cpu_paddr = (unsigned int)NULL;
 537         cpu_data_t                      *cpu_data_ptr = getCpuDatap();
 538
 539         if (from_boot) {
 540                 unsigned int    jtag = 0;
 541                 unsigned int    wfi;
 542
 543
 544                 if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
 545                         if (jtag != 0) {
 546                                 idle_enable = FALSE;
 547                         } else {
 548                                 idle_enable = TRUE;
 549                         }
 550                 } else {
 551                         idle_enable = TRUE;
 552                 }
 553
 554                 if (!PE_parse_boot_argn("wfi", &wfi, sizeof(wfi))) {
 555                         wfi = 1;
 556                 }
 557
 558                 if (wfi == 0) {
 559                         bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&patch_to_nop),
 560                             (addr64_t)ml_static_vtop((vm_offset_t)&wfi_inst), sizeof(unsigned));
 561                 }
 562                 if (wfi == 2) {
 563                         wfi_fast = 0;
 564                 }
 565
 566                 LowExceptionVectorsAddr = (void *)ml_io_map(ml_vtophys((vm_offset_t)gPhysBase), PAGE_SIZE);
 567
 568                 /* Copy Exception Vectors low, but don't touch the sleep token */
 569                 bcopy((void *)&ExceptionLowVectorsBase, (void *)LowExceptionVectorsAddr, 0x90);
 570                 bcopy(((void *)(((vm_offset_t)&ExceptionLowVectorsBase) + 0xA0)), ((void *)(((vm_offset_t)LowExceptionVectorsAddr) + 0xA0)), ARM_PGBYTES - 0xA0);
 571
 572                 start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
 573
 574                 BootArgs_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)BootArgs);
 575                 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&BootArgs_paddr),
 576                     (addr64_t)((unsigned int)(gPhysBase) +
 577                     ((unsigned int)&(ResetHandlerData.boot_args) - (unsigned int)&ExceptionLowVectorsBase)),
 578                     4);
 579
 580                 CpuDataEntries_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)CpuDataEntries);
 581                 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&CpuDataEntries_paddr),
 582                     (addr64_t)((unsigned int)(gPhysBase) +
 583                     ((unsigned int)&(ResetHandlerData.cpu_data_entries) - (unsigned int)&ExceptionLowVectorsBase)),
 584                     4);
 585
 586                 CleanPoC_DcacheRegion((vm_offset_t) phystokv(gPhysBase), PAGE_SIZE);
 587
 588                 resume_idle_cpu_paddr = (unsigned int)ml_static_vtop((vm_offset_t)&resume_idle_cpu);
 589         }
 590
 591         if (cpu_data_ptr == &BootCpuData) {
 592                 bcopy(((const void *)running_signature), (void *)(IOS_STATE), IOS_STATE_SIZE);
 593         }
 594         ;
 595
 596         cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
 597         clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
 598 }
 599
 600 void
 601 machine_track_platform_idle(boolean_t entry)
 602 {
 603         if (entry) {
 604                 os_atomic_inc(&cpu_idle_count, relaxed);
 605         } else {
 606                 os_atomic_dec(&cpu_idle_count, relaxed);
 607         }
 608 }