[apple/xnu.git] / osfmk / arm / cpu.c

/*
 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 *	File:	arm/cpu.c
 *
 *	cpu specific routines
 */

#include <kern/kalloc.h>
#include <kern/machine.h>
#include <kern/cpu_number.h>
#include <kern/percpu.h>
#include <kern/thread.h>
#include <kern/timer_queue.h>
#include <arm/cpu_data.h>
#include <arm/cpuid.h>
#include <arm/caches_internal.h>
#include <arm/cpu_data_internal.h>
#include <arm/cpu_internal.h>
#include <arm/misc_protos.h>
#include <arm/machine_cpu.h>
#include <arm/rtclock.h>
#include <arm/proc_reg.h>
#include <mach/processor_info.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <pexpert/arm/board_config.h>
#include <pexpert/arm/protos.h>
#include <sys/kdebug.h>

#include <machine/atomic.h>

#if KPC
#include <kern/kpc.h>
#endif

extern unsigned int resume_idle_cpu;
extern unsigned int start_cpu;

unsigned int   start_cpu_paddr;

extern boolean_t        idle_enable;
extern unsigned int     real_ncpus;
extern uint64_t         wake_abstime;

extern void* wfi_inst;
unsigned wfi_fast = 1;
unsigned patch_to_nop = 0xe1a00000;

void    *LowExceptionVectorsAddr;
#define IOS_STATE               (((vm_offset_t)LowExceptionVectorsAddr + 0x80))
#define IOS_STATE_SIZE  (0x08UL)
static const uint8_t suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
static const uint8_t running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};

/*
 *	Routine:	cpu_bootstrap
 *	Function:
 */
void
cpu_bootstrap(void)
{
}


/*
 *	Routine:	cpu_sleep
 *	Function:
 */
void
cpu_sleep(void)
{
	cpu_data_t     *cpu_data_ptr = getCpuDatap();
	pmap_switch_user_ttb(kernel_pmap);
	cpu_data_ptr->cpu_active_thread = current_thread();
	cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
	cpu_data_ptr->cpu_flags |= SleepState;
	cpu_data_ptr->cpu_user_debug = NULL;

	CleanPoC_Dcache();

	PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
}

_Atomic uint32_t cpu_idle_count = 0;

/*
 *	Routine:	cpu_idle
 *	Function:
 */
void __attribute__((noreturn))
cpu_idle(void)
{
	cpu_data_t     *cpu_data_ptr = getCpuDatap();
	uint64_t        new_idle_timeout_ticks = 0x0ULL, lastPop;

	if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) {
		Idle_load_context();
	}
	if (!SetIdlePop()) {
		Idle_load_context();
	}
	lastPop = cpu_data_ptr->rtcPop;

	pmap_switch_user_ttb(kernel_pmap);
	cpu_data_ptr->cpu_active_thread = current_thread();
	if (cpu_data_ptr->cpu_user_debug) {
		arm_debug_set(NULL);
	}
	cpu_data_ptr->cpu_user_debug = NULL;

	if (cpu_data_ptr->cpu_idle_notify != NULL) {
		cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
	}

	if (cpu_data_ptr->idle_timer_notify != 0) {
		if (new_idle_timeout_ticks == 0x0ULL) {
			/* turn off the idle timer */
			cpu_data_ptr->idle_timer_deadline = 0x0ULL;
		} else {
			/* set the new idle timeout */
			clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
		}
		timer_resync_deadlines();
		if (cpu_data_ptr->rtcPop != lastPop) {
			/*
			 * Ignore the return value here: this CPU has called idle_notify and
			 * committed to going idle.
			 */
			SetIdlePop();
		}
	}

#if KPC
	kpc_idle();
#endif /* KPC */

	platform_cache_idle_enter();
	cpu_idle_wfi((boolean_t) wfi_fast);
	platform_cache_idle_exit();

	ClearIdlePop(TRUE);
	cpu_idle_exit(FALSE);
}

/*
 *	Routine:	cpu_idle_exit
 *	Function:
 */
void
cpu_idle_exit(boolean_t from_reset __unused)
{
	uint64_t        new_idle_timeout_ticks = 0x0ULL;
	cpu_data_t     *cpu_data_ptr = getCpuDatap();

#if KPC
	kpc_idle_exit();
#endif


	pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread());

	if (cpu_data_ptr->cpu_idle_notify != NULL) {
		cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
	}

	if (cpu_data_ptr->idle_timer_notify != 0) {
		if (new_idle_timeout_ticks == 0x0ULL) {
			/* turn off the idle timer */
			cpu_data_ptr->idle_timer_deadline = 0x0ULL;
		} else {
			/* set the new idle timeout */
			clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
		}
		timer_resync_deadlines();
	}

	Idle_load_context();
}

void
cpu_init(void)
{
	cpu_data_t     *cdp = getCpuDatap();
	arm_cpu_info_t *cpu_info_p;

	if (cdp->cpu_type != CPU_TYPE_ARM) {
		cdp->cpu_type = CPU_TYPE_ARM;

		timer_call_queue_init(&cdp->rtclock_timer.queue);
		cdp->rtclock_timer.deadline = EndOfAllTime;

		if (cdp == &BootCpuData) {
			do_cpuid();
			do_cacheid();
			do_mvfpid();
		} else {
			/*
			 * We initialize non-boot CPUs here; the boot CPU is
			 * dealt with as part of pmap_bootstrap.
			 */
			pmap_cpu_data_init();
		}
		/* ARM_SMP: Assuming identical cpu */
		do_debugid();

		cpu_info_p = cpuid_info();

		/* switch based on CPU's reported architecture */
		switch (cpu_info_p->arm_info.arm_arch) {
		case CPU_ARCH_ARMv4T:
		case CPU_ARCH_ARMv5T:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V4T;
			break;
		case CPU_ARCH_ARMv5TE:
		case CPU_ARCH_ARMv5TEJ:
			if (cpu_info_p->arm_info.arm_implementor == CPU_VID_INTEL) {
				cdp->cpu_subtype = CPU_SUBTYPE_ARM_XSCALE;
			} else {
				cdp->cpu_subtype = CPU_SUBTYPE_ARM_V5TEJ;
			}
			break;
		case CPU_ARCH_ARMv6:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V6;
			break;
		case CPU_ARCH_ARMv7:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7;
			break;
		case CPU_ARCH_ARMv7f:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7F;
			break;
		case CPU_ARCH_ARMv7s:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7S;
			break;
		case CPU_ARCH_ARMv7k:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7K;
			break;
		default:
			cdp->cpu_subtype = CPU_SUBTYPE_ARM_ALL;
			break;
		}

		cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
	}
	cdp->cpu_stat.irq_ex_cnt_wake = 0;
	cdp->cpu_stat.ipi_cnt_wake = 0;
	cdp->cpu_running = TRUE;
	cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
	cdp->cpu_sleep_token = 0x0UL;
}

void
cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
{
	vm_offset_t             irq_stack = 0;
	vm_offset_t             fiq_stack = 0;

	kern_return_t kr = kernel_memory_allocate(kernel_map, &irq_stack,
	    INTSTACK_SIZE + (2 * PAGE_SIZE),
	    PAGE_MASK,
	    KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
	    VM_KERN_MEMORY_STACK);
	if (kr != KERN_SUCCESS) {
		panic("Unable to allocate cpu interrupt stack\n");
	}

	cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
	cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;

	kr = kernel_memory_allocate(kernel_map, &fiq_stack,
	    FIQSTACK_SIZE + (2 * PAGE_SIZE),
	    PAGE_MASK,
	    KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
	    VM_KERN_MEMORY_STACK);
	if (kr != KERN_SUCCESS) {
		panic("Unable to allocate cpu exception stack\n");
	}

	cpu_data_ptr->fiqstack_top = fiq_stack + PAGE_SIZE + FIQSTACK_SIZE;
	cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top;
}

void
cpu_data_free(cpu_data_t *cpu_data_ptr)
{
	if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
		return;
	}

	int cpu_number = cpu_data_ptr->cpu_number;

	if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
		OSDecrementAtomic((SInt32*)&real_ncpus);
		CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
		CpuDataEntries[cpu_number].cpu_data_paddr = 0;
		__builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
	}
	(kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
	(kfree)((void *)(cpu_data_ptr->fiqstack_top - FIQSTACK_SIZE), FIQSTACK_SIZE);
}

void
cpu_data_init(cpu_data_t *cpu_data_ptr)
{
	cpu_data_ptr->cpu_flags = 0;
#if     __arm__
	cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
#endif
	cpu_data_ptr->cpu_int_state = 0;
	cpu_data_ptr->cpu_pending_ast = AST_NONE;
	cpu_data_ptr->cpu_cache_dispatch = NULL;
	cpu_data_ptr->rtcPop = EndOfAllTime;
	cpu_data_ptr->rtclock_datap = &RTClockData;
	cpu_data_ptr->cpu_user_debug = NULL;
	cpu_data_ptr->cpu_base_timebase_low = 0;
	cpu_data_ptr->cpu_base_timebase_high = 0;
	cpu_data_ptr->cpu_idle_notify = NULL;
	cpu_data_ptr->cpu_idle_latency = 0x0ULL;
	cpu_data_ptr->cpu_idle_pop = 0x0ULL;
	cpu_data_ptr->cpu_reset_type = 0x0UL;
	cpu_data_ptr->cpu_reset_handler = 0x0UL;
	cpu_data_ptr->cpu_reset_assist = 0x0UL;
	cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
	cpu_data_ptr->cpu_phys_id = 0x0UL;
	cpu_data_ptr->cpu_l2_access_penalty = 0;
	cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
	cpu_data_ptr->cpu_cluster_id = 0;
	cpu_data_ptr->cpu_l2_id = 0;
	cpu_data_ptr->cpu_l2_size = 0;
	cpu_data_ptr->cpu_l3_id = 0;
	cpu_data_ptr->cpu_l3_size = 0;

	cpu_data_ptr->cpu_signal = SIGPdisabled;

	cpu_data_ptr->cpu_get_fiq_handler = NULL;
	cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
	cpu_data_ptr->cpu_tbd_hardware_val = NULL;
	cpu_data_ptr->cpu_get_decrementer_func = NULL;
	cpu_data_ptr->cpu_set_decrementer_func = NULL;
	cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
	cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
	cpu_data_ptr->cpu_xcall_p0 = NULL;
	cpu_data_ptr->cpu_xcall_p1 = NULL;
	cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
	cpu_data_ptr->cpu_imm_xcall_p1 = NULL;

#if     defined(ARMA7)
	cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL;
	cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL;
	cpu_data_ptr->cpu_CLWClean_req = 0x0ULL;
	cpu_data_ptr->cpu_CLWClean_last = 0x0ULL;
	cpu_data_ptr->cpu_CLW_active = 0x1UL;
#endif

#if !XNU_MONITOR
	pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;

	pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL;
	pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0;
	pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;

	bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
#endif
	cpu_data_ptr->halt_status = CPU_NOT_HALTED;
}

kern_return_t
cpu_data_register(cpu_data_t *cpu_data_ptr)
{
	int cpu;

	cpu = OSIncrementAtomic((SInt32*)&real_ncpus);
	if (real_ncpus > ml_get_cpu_count()) {
		return KERN_FAILURE;
	}

	cpu_data_ptr->cpu_number = cpu;
	__builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
	CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
	CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
	return KERN_SUCCESS;
}

kern_return_t
cpu_start(int cpu)
{
	kprintf("cpu_start() cpu: %d\n", cpu);
	if (cpu == cpu_number()) {
		cpu_machine_init();
		return KERN_SUCCESS;
	} else {
		cpu_data_t      *cpu_data_ptr;
		thread_t        first_thread;
		processor_t     processor;

		cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
		cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;

#if !XNU_MONITOR
		cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL;
#endif

		processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
		if (processor->startup_thread != THREAD_NULL) {
			first_thread = processor->startup_thread;
		} else {
			first_thread = processor->idle_thread;
		}
		cpu_data_ptr->cpu_active_thread = first_thread;
		first_thread->machine.CpuDatap = cpu_data_ptr;
		first_thread->machine.pcpu_data_base =
		    (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);

		flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
		flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
		(void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
		return KERN_SUCCESS;
	}
}

void
cpu_timebase_init(boolean_t from_boot __unused)
{
	cpu_data_t *cdp = getCpuDatap();

	if (cdp->cpu_get_fiq_handler == NULL) {
		cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
		cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
		cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
		cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
		cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
	}
	cdp->cpu_decrementer = 0x7FFFFFFFUL;
	cdp->cpu_timebase_low = 0x0UL;
	cdp->cpu_timebase_high = 0x0UL;

#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
	/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
	 * are 32-bit. */
	cdp->cpu_base_timebase_low = rtclock_base_abstime_low;
	cdp->cpu_base_timebase_high = rtclock_base_abstime_high;
#else
	*((uint64_t *) &cdp->cpu_base_timebase_low) = rtclock_base_abstime;
#endif
}


__attribute__((noreturn))
void
ml_arm_sleep(void)
{
	cpu_data_t     *cpu_data_ptr = getCpuDatap();

	if (cpu_data_ptr == &BootCpuData) {
		cpu_data_t      *target_cdp;
		unsigned int    cpu;

		const unsigned int max_cpu_id = ml_get_max_cpu_number();
		for (cpu = 0; cpu <= max_cpu_id; cpu++) {
			target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
			if (target_cdp == (cpu_data_t *)NULL) {
				break;
			}

			if (target_cdp == cpu_data_ptr) {
				continue;
			}

			while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
				;
			}
		}

		/* Now that the other cores have entered the sleep path, set
		 * the abstime fixup we'll use when we resume.*/
		rtclock_base_abstime = ml_get_timebase();
		wake_abstime = rtclock_base_abstime;
	} else {
		platform_cache_disable();
		CleanPoU_Dcache();
	}
	cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
#if     defined(ARMA7)
	cpu_data_ptr->cpu_CLWFlush_req = 0;
	cpu_data_ptr->cpu_CLWClean_req = 0;
	__builtin_arm_dmb(DMB_ISH);
	cpu_data_ptr->cpu_CLW_active = 0;
#endif
	if (cpu_data_ptr == &BootCpuData) {
		platform_cache_disable();
		platform_cache_shutdown();
		bcopy((const void *)suspend_signature, (void *)(IOS_STATE), IOS_STATE_SIZE);
	} else {
		CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
	}

	__builtin_arm_dsb(DSB_SY);
	while (TRUE) {
#if     __ARM_ENABLE_WFE_
		__builtin_arm_wfe();
#endif
	} /* Spin */
}

void
cpu_machine_idle_init(boolean_t from_boot)
{
	static const unsigned int       *BootArgs_paddr = (unsigned int *)NULL;
	static const unsigned int       *CpuDataEntries_paddr = (unsigned int *)NULL;
	static unsigned int             resume_idle_cpu_paddr = (unsigned int)NULL;
	cpu_data_t                      *cpu_data_ptr = getCpuDatap();

	if (from_boot) {
		unsigned int    jtag = 0;
		unsigned int    wfi;


		if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
			if (jtag != 0) {
				idle_enable = FALSE;
			} else {
				idle_enable = TRUE;
			}
		} else {
			idle_enable = TRUE;
		}

		if (!PE_parse_boot_argn("wfi", &wfi, sizeof(wfi))) {
			wfi = 1;
		}

		if (wfi == 0) {
			bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&patch_to_nop),
			    (addr64_t)ml_static_vtop((vm_offset_t)&wfi_inst), sizeof(unsigned));
		}
		if (wfi == 2) {
			wfi_fast = 0;
		}

		LowExceptionVectorsAddr = (void *)ml_io_map(ml_vtophys((vm_offset_t)gPhysBase), PAGE_SIZE);

		/* Copy Exception Vectors low, but don't touch the sleep token */
		bcopy((void *)&ExceptionLowVectorsBase, (void *)LowExceptionVectorsAddr, 0x90);
		bcopy(((void *)(((vm_offset_t)&ExceptionLowVectorsBase) + 0xA0)), ((void *)(((vm_offset_t)LowExceptionVectorsAddr) + 0xA0)), ARM_PGBYTES - 0xA0);

		start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);

		BootArgs_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)BootArgs);
		bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&BootArgs_paddr),
		    (addr64_t)((unsigned int)(gPhysBase) +
		    ((unsigned int)&(ResetHandlerData.boot_args) - (unsigned int)&ExceptionLowVectorsBase)),
		    4);

		CpuDataEntries_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)CpuDataEntries);
		bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&CpuDataEntries_paddr),
		    (addr64_t)((unsigned int)(gPhysBase) +
		    ((unsigned int)&(ResetHandlerData.cpu_data_entries) - (unsigned int)&ExceptionLowVectorsBase)),
		    4);

		CleanPoC_DcacheRegion((vm_offset_t) phystokv(gPhysBase), PAGE_SIZE);

		resume_idle_cpu_paddr = (unsigned int)ml_static_vtop((vm_offset_t)&resume_idle_cpu);
	}

	if (cpu_data_ptr == &BootCpuData) {
		bcopy(((const void *)running_signature), (void *)(IOS_STATE), IOS_STATE_SIZE);
	}
	;

	cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
	clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
}

void
machine_track_platform_idle(boolean_t entry)
{
	if (entry) {
		os_atomic_inc(&cpu_idle_count, relaxed);
	} else {
		os_atomic_dec(&cpu_idle_count, relaxed);
	}
}
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007-2016 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	/*
	29	* File: arm/cpu.c
	30	*
	31	* cpu specific routines
	32	*/
	33
	34	#include <kern/kalloc.h>
	35	#include <kern/machine.h>
	36	#include <kern/cpu_number.h>
f427ee49	37	#include <kern/percpu.h>
5ba3f43e A	38	#include <kern/thread.h>
	39	#include <kern/timer_queue.h>
	40	#include <arm/cpu_data.h>
	41	#include <arm/cpuid.h>
	42	#include <arm/caches_internal.h>
	43	#include <arm/cpu_data_internal.h>
	44	#include <arm/cpu_internal.h>
	45	#include <arm/misc_protos.h>
	46	#include <arm/machine_cpu.h>
	47	#include <arm/rtclock.h>
	48	#include <arm/proc_reg.h>
	49	#include <mach/processor_info.h>
	50	#include <vm/pmap.h>
	51	#include <vm/vm_kern.h>
	52	#include <vm/vm_map.h>
	53	#include <pexpert/arm/board_config.h>
	54	#include <pexpert/arm/protos.h>
	55	#include <sys/kdebug.h>
	56
	57	#include <machine/atomic.h>
	58
	59	#if KPC
	60	#include <kern/kpc.h>
	61	#endif
	62
	63	extern unsigned int resume_idle_cpu;
	64	extern unsigned int start_cpu;
	65
	66	unsigned int start_cpu_paddr;
	67
0a7de745 A	68	extern boolean_t idle_enable;
	69	extern unsigned int real_ncpus;
	70	extern uint64_t wake_abstime;
5ba3f43e A	71
	72	extern void* wfi_inst;
	73	unsigned wfi_fast = 1;
	74	unsigned patch_to_nop = 0xe1a00000;
	75
0a7de745 A	76	void *LowExceptionVectorsAddr;
	77	#define IOS_STATE (((vm_offset_t)LowExceptionVectorsAddr + 0x80))
	78	#define IOS_STATE_SIZE (0x08UL)
5ba3f43e A	79	static const uint8_t suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
	80	static const uint8_t running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
	81
	82	/*
	83	* Routine: cpu_bootstrap
	84	* Function:
	85	*/
	86	void
	87	cpu_bootstrap(void)
	88	{
	89	}
	90
	91
	92	/*
	93	* Routine: cpu_sleep
	94	* Function:
	95	*/
	96	void
	97	cpu_sleep(void)
	98	{
	99	cpu_data_t *cpu_data_ptr = getCpuDatap();
	100	pmap_switch_user_ttb(kernel_pmap);
	101	cpu_data_ptr->cpu_active_thread = current_thread();
	102	cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
	103	cpu_data_ptr->cpu_flags \|= SleepState;
	104	cpu_data_ptr->cpu_user_debug = NULL;
	105
	106	CleanPoC_Dcache();
	107
	108	PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
5ba3f43e A	109	}
	110
	111	_Atomic uint32_t cpu_idle_count = 0;
	112
	113	/*
	114	* Routine: cpu_idle
	115	* Function:
	116	*/
	117	void __attribute__((noreturn))
	118	cpu_idle(void)
	119	{
	120	cpu_data_t *cpu_data_ptr = getCpuDatap();
0a7de745	121	uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
5ba3f43e	122
0a7de745	123	if ((!idle_enable) \|\| (cpu_data_ptr->cpu_signal & SIGPdisabled)) {
5ba3f43e	124	Idle_load_context();
0a7de745 A	125	}
0a7de745 A	126	if (!SetIdlePop()) {
5ba3f43e	127	Idle_load_context();
0a7de745	128	}
5ba3f43e A	129	lastPop = cpu_data_ptr->rtcPop;
	130
	131	pmap_switch_user_ttb(kernel_pmap);
	132	cpu_data_ptr->cpu_active_thread = current_thread();
0a7de745	133	if (cpu_data_ptr->cpu_user_debug) {
5ba3f43e	134	arm_debug_set(NULL);
0a7de745	135	}
5ba3f43e A	136	cpu_data_ptr->cpu_user_debug = NULL;
5ba3f43e A	137
f427ee49 A	138	if (cpu_data_ptr->cpu_idle_notify != NULL) {
f427ee49 A	139	cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
0a7de745	140	}
5ba3f43e A	141
	142	if (cpu_data_ptr->idle_timer_notify != 0) {
	143	if (new_idle_timeout_ticks == 0x0ULL) {
	144	/* turn off the idle timer */
	145	cpu_data_ptr->idle_timer_deadline = 0x0ULL;
	146	} else {
	147	/* set the new idle timeout */
	148	clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
	149	}
	150	timer_resync_deadlines();
0a7de745	151	if (cpu_data_ptr->rtcPop != lastPop) {
f427ee49 A	152	/*
	153	* Ignore the return value here: this CPU has called idle_notify and
	154	* committed to going idle.
	155	*/
5ba3f43e	156	SetIdlePop();
0a7de745	157	}
5ba3f43e A	158	}
	159
	160	#if KPC
	161	kpc_idle();
f427ee49	162	#endif /* KPC */
5ba3f43e A	163
	164	platform_cache_idle_enter();
	165	cpu_idle_wfi((boolean_t) wfi_fast);
	166	platform_cache_idle_exit();
	167
	168	ClearIdlePop(TRUE);
d9a64523	169	cpu_idle_exit(FALSE);
5ba3f43e A	170	}
	171
	172	/*
	173	* Routine: cpu_idle_exit
	174	* Function:
	175	*/
	176	void
d9a64523	177	cpu_idle_exit(boolean_t from_reset __unused)
5ba3f43e	178	{
0a7de745	179	uint64_t new_idle_timeout_ticks = 0x0ULL;
5ba3f43e A	180	cpu_data_t *cpu_data_ptr = getCpuDatap();
	181
	182	#if KPC
	183	kpc_idle_exit();
	184	#endif
	185
	186
	187	pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread());
	188
f427ee49 A	189	if (cpu_data_ptr->cpu_idle_notify != NULL) {
f427ee49 A	190	cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
0a7de745	191	}
5ba3f43e A	192
	193	if (cpu_data_ptr->idle_timer_notify != 0) {
	194	if (new_idle_timeout_ticks == 0x0ULL) {
	195	/* turn off the idle timer */
	196	cpu_data_ptr->idle_timer_deadline = 0x0ULL;
	197	} else {
	198	/* set the new idle timeout */
	199	clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
	200	}
	201	timer_resync_deadlines();
	202	}
	203
	204	Idle_load_context();
	205	}
	206
	207	void
	208	cpu_init(void)
	209	{
	210	cpu_data_t *cdp = getCpuDatap();
	211	arm_cpu_info_t *cpu_info_p;
	212
	213	if (cdp->cpu_type != CPU_TYPE_ARM) {
5ba3f43e A	214	cdp->cpu_type = CPU_TYPE_ARM;
	215
	216	timer_call_queue_init(&cdp->rtclock_timer.queue);
	217	cdp->rtclock_timer.deadline = EndOfAllTime;
	218
	219	if (cdp == &BootCpuData) {
	220	do_cpuid();
	221	do_cacheid();
	222	do_mvfpid();
	223	} else {
	224	/*
	225	* We initialize non-boot CPUs here; the boot CPU is
	226	* dealt with as part of pmap_bootstrap.
	227	*/
	228	pmap_cpu_data_init();
	229	}
	230	/* ARM_SMP: Assuming identical cpu */
	231	do_debugid();
	232
	233	cpu_info_p = cpuid_info();
	234
	235	/* switch based on CPU's reported architecture */
	236	switch (cpu_info_p->arm_info.arm_arch) {
	237	case CPU_ARCH_ARMv4T:
	238	case CPU_ARCH_ARMv5T:
	239	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V4T;
	240	break;
	241	case CPU_ARCH_ARMv5TE:
	242	case CPU_ARCH_ARMv5TEJ:
0a7de745	243	if (cpu_info_p->arm_info.arm_implementor == CPU_VID_INTEL) {
5ba3f43e	244	cdp->cpu_subtype = CPU_SUBTYPE_ARM_XSCALE;
0a7de745	245	} else {
5ba3f43e	246	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V5TEJ;
0a7de745	247	}
5ba3f43e A	248	break;
	249	case CPU_ARCH_ARMv6:
	250	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V6;
	251	break;
	252	case CPU_ARCH_ARMv7:
	253	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7;
	254	break;
	255	case CPU_ARCH_ARMv7f:
	256	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7F;
	257	break;
	258	case CPU_ARCH_ARMv7s:
	259	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7S;
	260	break;
	261	case CPU_ARCH_ARMv7k:
	262	cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7K;
	263	break;
	264	default:
	265	cdp->cpu_subtype = CPU_SUBTYPE_ARM_ALL;
	266	break;
	267	}
	268
	269	cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
	270	}
	271	cdp->cpu_stat.irq_ex_cnt_wake = 0;
	272	cdp->cpu_stat.ipi_cnt_wake = 0;
5ba3f43e A	273	cdp->cpu_running = TRUE;
	274	cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
	275	cdp->cpu_sleep_token = 0x0UL;
5ba3f43e A	276	}
5ba3f43e A	277
d9a64523 A	278	void
d9a64523 A	279	cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
5ba3f43e	280	{
0a7de745 A	281	vm_offset_t irq_stack = 0;
0a7de745 A	282	vm_offset_t fiq_stack = 0;
d9a64523 A	283
d9a64523 A	284	kern_return_t kr = kernel_memory_allocate(kernel_map, &irq_stack,
0a7de745 A	285	INTSTACK_SIZE + (2 * PAGE_SIZE),
	286	PAGE_MASK,
	287	KMA_GUARD_FIRST \| KMA_GUARD_LAST \| KMA_KSTACK \| KMA_KOBJECT,
	288	VM_KERN_MEMORY_STACK);
	289	if (kr != KERN_SUCCESS) {
d9a64523	290	panic("Unable to allocate cpu interrupt stack\n");
0a7de745	291	}
d9a64523 A	292
	293	cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
	294	cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;
	295
	296	kr = kernel_memory_allocate(kernel_map, &fiq_stack,
0a7de745 A	297	FIQSTACK_SIZE + (2 * PAGE_SIZE),
	298	PAGE_MASK,
	299	KMA_GUARD_FIRST \| KMA_GUARD_LAST \| KMA_KSTACK \| KMA_KOBJECT,
	300	VM_KERN_MEMORY_STACK);
	301	if (kr != KERN_SUCCESS) {
d9a64523	302	panic("Unable to allocate cpu exception stack\n");
0a7de745	303	}
d9a64523 A	304
	305	cpu_data_ptr->fiqstack_top = fiq_stack + PAGE_SIZE + FIQSTACK_SIZE;
	306	cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top;
5ba3f43e A	307	}
5ba3f43e A	308
5ba3f43e A	309	void
	310	cpu_data_free(cpu_data_t *cpu_data_ptr)
	311	{
cb323159	312	if ((cpu_data_ptr == NULL) \|\| (cpu_data_ptr == &BootCpuData)) {
0a7de745 A	313	return;
0a7de745 A	314	}
5ba3f43e	315
f427ee49 A	316	int cpu_number = cpu_data_ptr->cpu_number;
	317
	318	if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
cb323159	319	OSDecrementAtomic((SInt32*)&real_ncpus);
f427ee49 A	320	CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
f427ee49 A	321	CpuDataEntries[cpu_number].cpu_data_paddr = 0;
cb323159 A	322	__builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
cb323159 A	323	}
0a7de745 A	324	(kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
0a7de745 A	325	(kfree)((void *)(cpu_data_ptr->fiqstack_top - FIQSTACK_SIZE), FIQSTACK_SIZE);
5ba3f43e A	326	}
	327
	328	void
	329	cpu_data_init(cpu_data_t *cpu_data_ptr)
	330	{
5ba3f43e	331	cpu_data_ptr->cpu_flags = 0;
0a7de745	332	#if __arm__
5ba3f43e A	333	cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
5ba3f43e A	334	#endif
5ba3f43e A	335	cpu_data_ptr->cpu_int_state = 0;
5ba3f43e A	336	cpu_data_ptr->cpu_pending_ast = AST_NONE;
f427ee49	337	cpu_data_ptr->cpu_cache_dispatch = NULL;
5ba3f43e A	338	cpu_data_ptr->rtcPop = EndOfAllTime;
	339	cpu_data_ptr->rtclock_datap = &RTClockData;
	340	cpu_data_ptr->cpu_user_debug = NULL;
	341	cpu_data_ptr->cpu_base_timebase_low = 0;
	342	cpu_data_ptr->cpu_base_timebase_high = 0;
f427ee49	343	cpu_data_ptr->cpu_idle_notify = NULL;
5ba3f43e A	344	cpu_data_ptr->cpu_idle_latency = 0x0ULL;
	345	cpu_data_ptr->cpu_idle_pop = 0x0ULL;
	346	cpu_data_ptr->cpu_reset_type = 0x0UL;
	347	cpu_data_ptr->cpu_reset_handler = 0x0UL;
	348	cpu_data_ptr->cpu_reset_assist = 0x0UL;
	349	cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
	350	cpu_data_ptr->cpu_phys_id = 0x0UL;
	351	cpu_data_ptr->cpu_l2_access_penalty = 0;
	352	cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
	353	cpu_data_ptr->cpu_cluster_id = 0;
	354	cpu_data_ptr->cpu_l2_id = 0;
	355	cpu_data_ptr->cpu_l2_size = 0;
	356	cpu_data_ptr->cpu_l3_id = 0;
	357	cpu_data_ptr->cpu_l3_size = 0;
	358
	359	cpu_data_ptr->cpu_signal = SIGPdisabled;
	360
5ba3f43e A	361	cpu_data_ptr->cpu_get_fiq_handler = NULL;
	362	cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
	363	cpu_data_ptr->cpu_tbd_hardware_val = NULL;
	364	cpu_data_ptr->cpu_get_decrementer_func = NULL;
	365	cpu_data_ptr->cpu_set_decrementer_func = NULL;
	366	cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
	367	cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
	368	cpu_data_ptr->cpu_xcall_p0 = NULL;
	369	cpu_data_ptr->cpu_xcall_p1 = NULL;
cb323159 A	370	cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
cb323159 A	371	cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
5ba3f43e	372
f427ee49	373	#if defined(ARMA7)
5ba3f43e A	374	cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL;
	375	cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL;
	376	cpu_data_ptr->cpu_CLWClean_req = 0x0ULL;
	377	cpu_data_ptr->cpu_CLWClean_last = 0x0ULL;
	378	cpu_data_ptr->cpu_CLW_active = 0x1UL;
	379	#endif
	380
c6bf4f31	381	#if !XNU_MONITOR
5ba3f43e A	382	pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
	383
	384	pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL;
	385	pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0;
	386	pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
	387
f427ee49	388	bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
c6bf4f31	389	#endif
5ba3f43e A	390	cpu_data_ptr->halt_status = CPU_NOT_HALTED;
	391	}
	392
	393	kern_return_t
	394	cpu_data_register(cpu_data_t *cpu_data_ptr)
	395	{
	396	int cpu;
	397
	398	cpu = OSIncrementAtomic((SInt32*)&real_ncpus);
f427ee49	399	if (real_ncpus > ml_get_cpu_count()) {
5ba3f43e A	400	return KERN_FAILURE;
	401	}
	402
	403	cpu_data_ptr->cpu_number = cpu;
cb323159	404	__builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
5ba3f43e	405	CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
0a7de745	406	CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
5ba3f43e A	407	return KERN_SUCCESS;
	408	}
	409
	410	kern_return_t
	411	cpu_start(int cpu)
	412	{
	413	kprintf("cpu_start() cpu: %d\n", cpu);
	414	if (cpu == cpu_number()) {
	415	cpu_machine_init();
	416	return KERN_SUCCESS;
	417	} else {
0a7de745 A	418	cpu_data_t *cpu_data_ptr;
0a7de745 A	419	thread_t first_thread;
f427ee49	420	processor_t processor;
5ba3f43e A	421
	422	cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
	423	cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
	424
c6bf4f31	425	#if !XNU_MONITOR
5ba3f43e	426	cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL;
c6bf4f31	427	#endif
5ba3f43e	428
f427ee49 A	429	processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
	430	if (processor->startup_thread != THREAD_NULL) {
	431	first_thread = processor->startup_thread;
0a7de745	432	} else {
f427ee49	433	first_thread = processor->idle_thread;
0a7de745	434	}
5ba3f43e A	435	cpu_data_ptr->cpu_active_thread = first_thread;
5ba3f43e A	436	first_thread->machine.CpuDatap = cpu_data_ptr;
f427ee49 A	437	first_thread->machine.pcpu_data_base =
f427ee49 A	438	(vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
5ba3f43e A	439
	440	flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
	441	flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
	442	(void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
	443	return KERN_SUCCESS;
5ba3f43e A	444	}
	445	}
	446
	447	void
	448	cpu_timebase_init(boolean_t from_boot __unused)
	449	{
	450	cpu_data_t *cdp = getCpuDatap();
	451
	452	if (cdp->cpu_get_fiq_handler == NULL) {
	453	cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
	454	cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
	455	cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
	456	cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
	457	cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
	458	}
	459	cdp->cpu_decrementer = 0x7FFFFFFFUL;
	460	cdp->cpu_timebase_low = 0x0UL;
	461	cdp->cpu_timebase_high = 0x0UL;
	462
	463	#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
	464	/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
	465	* are 32-bit. */
	466	cdp->cpu_base_timebase_low = rtclock_base_abstime_low;
	467	cdp->cpu_base_timebase_high = rtclock_base_abstime_high;
	468	#else
0a7de745	469	((uint64_t ) &cdp->cpu_base_timebase_low) = rtclock_base_abstime;
5ba3f43e A	470	#endif
	471	}
	472
	473
	474	__attribute__((noreturn))
	475	void
	476	ml_arm_sleep(void)
	477	{
	478	cpu_data_t *cpu_data_ptr = getCpuDatap();
	479
	480	if (cpu_data_ptr == &BootCpuData) {
0a7de745 A	481	cpu_data_t *target_cdp;
0a7de745 A	482	unsigned int cpu;
5ba3f43e	483
f427ee49 A	484	const unsigned int max_cpu_id = ml_get_max_cpu_number();
f427ee49 A	485	for (cpu = 0; cpu <= max_cpu_id; cpu++) {
5ba3f43e	486	target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
0a7de745	487	if (target_cdp == (cpu_data_t *)NULL) {
5ba3f43e	488	break;
0a7de745	489	}
5ba3f43e	490
0a7de745	491	if (target_cdp == cpu_data_ptr) {
5ba3f43e	492	continue;
0a7de745	493	}
5ba3f43e	494
0a7de745 A	495	while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
	496	;
	497	}
5ba3f43e A	498	}
	499
	500	/* Now that the other cores have entered the sleep path, set
	501	* the abstime fixup we'll use when we resume.*/
	502	rtclock_base_abstime = ml_get_timebase();
	503	wake_abstime = rtclock_base_abstime;
5ba3f43e A	504	} else {
	505	platform_cache_disable();
	506	CleanPoU_Dcache();
	507	}
	508	cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
f427ee49	509	#if defined(ARMA7)
5ba3f43e A	510	cpu_data_ptr->cpu_CLWFlush_req = 0;
	511	cpu_data_ptr->cpu_CLWClean_req = 0;
	512	__builtin_arm_dmb(DMB_ISH);
	513	cpu_data_ptr->cpu_CLW_active = 0;
	514	#endif
	515	if (cpu_data_ptr == &BootCpuData) {
	516	platform_cache_disable();
	517	platform_cache_shutdown();
	518	bcopy((const void )suspend_signature, (void )(IOS_STATE), IOS_STATE_SIZE);
0a7de745	519	} else {
5ba3f43e	520	CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
0a7de745	521	}
5ba3f43e A	522
	523	__builtin_arm_dsb(DSB_SY);
	524	while (TRUE) {
	525	#if __ARM_ENABLE_WFE_
	526	__builtin_arm_wfe();
	527	#endif
	528	} /* Spin */
	529	}
	530
	531	void
	532	cpu_machine_idle_init(boolean_t from_boot)
	533	{
0a7de745 A	534	static const unsigned int BootArgs_paddr = (unsigned int )NULL;
	535	static const unsigned int CpuDataEntries_paddr = (unsigned int )NULL;
	536	static unsigned int resume_idle_cpu_paddr = (unsigned int)NULL;
	537	cpu_data_t *cpu_data_ptr = getCpuDatap();
5ba3f43e A	538
	539	if (from_boot) {
	540	unsigned int jtag = 0;
	541	unsigned int wfi;
	542
	543
0a7de745 A	544	if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
0a7de745 A	545	if (jtag != 0) {
5ba3f43e	546	idle_enable = FALSE;
0a7de745	547	} else {
5ba3f43e	548	idle_enable = TRUE;
0a7de745 A	549	}
0a7de745 A	550	} else {
5ba3f43e	551	idle_enable = TRUE;
0a7de745	552	}
5ba3f43e	553
0a7de745	554	if (!PE_parse_boot_argn("wfi", &wfi, sizeof(wfi))) {
5ba3f43e	555	wfi = 1;
0a7de745	556	}
5ba3f43e	557
0a7de745	558	if (wfi == 0) {
5ba3f43e	559	bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&patch_to_nop),
0a7de745 A	560	(addr64_t)ml_static_vtop((vm_offset_t)&wfi_inst), sizeof(unsigned));
	561	}
	562	if (wfi == 2) {
5ba3f43e	563	wfi_fast = 0;
0a7de745	564	}
5ba3f43e A	565
	566	LowExceptionVectorsAddr = (void *)ml_io_map(ml_vtophys((vm_offset_t)gPhysBase), PAGE_SIZE);
	567
	568	/* Copy Exception Vectors low, but don't touch the sleep token */
	569	bcopy((void )&ExceptionLowVectorsBase, (void )LowExceptionVectorsAddr, 0x90);
	570	bcopy(((void )(((vm_offset_t)&ExceptionLowVectorsBase) + 0xA0)), ((void )(((vm_offset_t)LowExceptionVectorsAddr) + 0xA0)), ARM_PGBYTES - 0xA0);
	571
	572	start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
	573
	574	BootArgs_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)BootArgs);
	575	bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&BootArgs_paddr),
0a7de745 A	576	(addr64_t)((unsigned int)(gPhysBase) +
	577	((unsigned int)&(ResetHandlerData.boot_args) - (unsigned int)&ExceptionLowVectorsBase)),
	578	4);
5ba3f43e A	579
	580	CpuDataEntries_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)CpuDataEntries);
	581	bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&CpuDataEntries_paddr),
0a7de745 A	582	(addr64_t)((unsigned int)(gPhysBase) +
	583	((unsigned int)&(ResetHandlerData.cpu_data_entries) - (unsigned int)&ExceptionLowVectorsBase)),
	584	4);
5ba3f43e	585
d9a64523	586	CleanPoC_DcacheRegion((vm_offset_t) phystokv(gPhysBase), PAGE_SIZE);
5ba3f43e A	587
5ba3f43e A	588	resume_idle_cpu_paddr = (unsigned int)ml_static_vtop((vm_offset_t)&resume_idle_cpu);
5ba3f43e A	589	}
	590
	591	if (cpu_data_ptr == &BootCpuData) {
	592	bcopy(((const void )running_signature), (void )(IOS_STATE), IOS_STATE_SIZE);
0a7de745 A	593	}
0a7de745 A	594	;
5ba3f43e A	595
	596	cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
	597	clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
	598	}
	599
	600	void
	601	machine_track_platform_idle(boolean_t entry)
	602	{
0a7de745	603	if (entry) {
cb323159	604	os_atomic_inc(&cpu_idle_count, relaxed);
0a7de745	605	} else {
cb323159	606	os_atomic_dec(&cpu_idle_count, relaxed);
0a7de745	607	}
5ba3f43e	608	}