X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4d15aeb193b2c68f1d38666c317f8d3734f5f083..5ba3f43ea354af8ad55bea84372a2bc834d8757c:/osfmk/arm/cpu_common.c diff --git a/osfmk/arm/cpu_common.c b/osfmk/arm/cpu_common.c new file mode 100644 index 000000000..ac41435d4 --- /dev/null +++ b/osfmk/arm/cpu_common.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm/cpu_common.c + * + * cpu routines common to all supported arm variants + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KPERF +void kperf_signal_handler(unsigned int cpu_number); +#endif + +struct processor BootProcessor; + +unsigned int real_ncpus = 1; +boolean_t idle_enable = FALSE; +uint64_t wake_abstime=0x0ULL; + + +cpu_data_t * +cpu_datap(int cpu) +{ + assert(cpu < MAX_CPUS); + return (CpuDataEntries[cpu].cpu_data_vaddr); +} + +kern_return_t +cpu_control(int slot_num, + processor_info_t info, + unsigned int count) +{ + printf("cpu_control(%d,%p,%d) not implemented\n", + slot_num, info, count); + return (KERN_FAILURE); +} + +kern_return_t +cpu_info_count(processor_flavor_t flavor, + unsigned int *count) +{ + + switch (flavor) { + case PROCESSOR_CPU_STAT: + *count = PROCESSOR_CPU_STAT_COUNT; + return (KERN_SUCCESS); + + default: + *count = 0; + return (KERN_FAILURE); + } +} + +kern_return_t +cpu_info(processor_flavor_t flavor, + int slot_num, + processor_info_t info, + unsigned int *count) +{ + switch (flavor) { + case PROCESSOR_CPU_STAT: + { + processor_cpu_stat_t cpu_stat; + cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr; + + if (*count < PROCESSOR_CPU_STAT_COUNT) + return (KERN_FAILURE); + + cpu_stat = (processor_cpu_stat_t) info; + cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt; + cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt; + cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt; + cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt; + cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt; + cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt; + cpu_stat->vfp_shortv_cnt = 0; + cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt; + cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt; + + *count = PROCESSOR_CPU_STAT_COUNT; + + return (KERN_SUCCESS); + } + + default: + return (KERN_FAILURE); + } +} + +/* + * Routine: cpu_doshutdown + * Function: + */ +void +cpu_doshutdown(void (*doshutdown) (processor_t), + processor_t processor) +{ + doshutdown(processor); +} + +/* + * Routine: cpu_idle_tickle + * + */ +void +cpu_idle_tickle(void) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + uint64_t new_idle_timeout_ticks = 0x0ULL; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr->idle_timer_notify != (void *)NULL) { + ((idle_timer_t)cpu_data_ptr->idle_timer_notify)(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); + if (new_idle_timeout_ticks != 0x0ULL) { + /* if a new idle timeout was requested set the new idle timer deadline */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } else { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } + timer_resync_deadlines(); + } + (void) ml_set_interrupts_enabled(intr); +} + +static void +cpu_handle_xcall(cpu_data_t *cpu_data_ptr) +{ + broadcastFunc xfunc; + void *xparam; + + __c11_atomic_thread_fence(memory_order_acquire_smp); + /* Come back around if cpu_signal_internal is running on another CPU and has just + * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/ + if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) { + xfunc = cpu_data_ptr->cpu_xcall_p0; + xparam = cpu_data_ptr->cpu_xcall_p1; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + __c11_atomic_thread_fence(memory_order_acq_rel_smp); + hw_atomic_and_noret(&cpu_data_ptr->cpu_signal, ~SIGPxcall); + xfunc(xparam); + } + +} + +unsigned int +cpu_broadcast_xcall(uint32_t *synch, + boolean_t self_xcall, + broadcastFunc func, + void *parm) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + cpu_data_t *target_cpu_datap; + unsigned int failsig; + int cpu; + int max_cpu; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + failsig = 0; + + if (synch != NULL) { + *synch = real_ncpus; + assert_wait((event_t)synch, THREAD_UNINT); + } + + max_cpu = ml_get_max_cpu_number(); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + + if(KERN_SUCCESS != cpu_signal(target_cpu_datap, SIGPxcall, (void *)func, parm)) { + failsig++; + } + } + + + if (self_xcall) { + func(parm); + } + + (void) ml_set_interrupts_enabled(intr); + + if (synch != NULL) { + if (hw_atomic_sub(synch, (!self_xcall)? failsig+1 : failsig) == 0) + clear_wait(current_thread(), THREAD_AWAKENED); + else + thread_block(THREAD_CONTINUE_NULL); + } + + if (!self_xcall) + return (real_ncpus - failsig - 1); + else + return (real_ncpus - failsig); +} + +kern_return_t +cpu_xcall(int cpu_number, broadcastFunc func, void *param) +{ + cpu_data_t *target_cpu_datap; + + if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) + return KERN_INVALID_ARGUMENT; + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr; + if (target_cpu_datap == NULL) + return KERN_INVALID_ARGUMENT; + + return cpu_signal(target_cpu_datap, SIGPxcall, (void*)func, param); +} + +static kern_return_t +cpu_signal_internal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1, + boolean_t defer) +{ + unsigned int Check_SIGPdisabled; + int current_signals; + Boolean swap_success; + boolean_t interruptible = ml_set_interrupts_enabled(FALSE); + cpu_data_t *current_proc = getCpuDatap(); + + /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */ + if (defer) { + assert(signal == SIGPnop); + } + + if (current_proc != target_proc) + Check_SIGPdisabled = SIGPdisabled; + else + Check_SIGPdisabled = 0; + + if (signal == SIGPxcall) { + do { + current_signals = target_proc->cpu_signal; + if ((current_signals & SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = SIGPxcall; + target_proc->failed_xcall = p0; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + swap_success = OSCompareAndSwap(current_signals & (~SIGPxcall), current_signals | SIGPxcall, + &target_proc->cpu_signal); + + /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn + * be trying to xcall us. Since we have interrupts disabled that can deadlock, + * so break the deadlock by draining pending xcalls. */ + if (!swap_success && (current_proc->cpu_signal & SIGPxcall)) + cpu_handle_xcall(current_proc); + + } while (!swap_success); + + target_proc->cpu_xcall_p0 = p0; + target_proc->cpu_xcall_p1 = p1; + } else { + do { + current_signals = target_proc->cpu_signal; + if ((Check_SIGPdisabled !=0 ) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = signal; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + + swap_success = OSCompareAndSwap(current_signals, current_signals | signal, + &target_proc->cpu_signal); + } while (!swap_success); + } + + /* + * Issue DSB here to guarantee: 1) prior stores to pending signal mask and xcall params + * will be visible to other cores when the IPI is dispatched, and 2) subsequent + * instructions to signal the other cores will not execute until after the barrier. + * DMB would be sufficient to guarantee 1) but not 2). + */ + __builtin_arm_dsb(DSB_ISH); + + if (!(target_proc->cpu_signal & SIGPdisabled)) { + if (defer) { + PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id); + } else { + PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id); + } + } + + ml_set_interrupts_enabled(interruptible); + return (KERN_SUCCESS); +} + +kern_return_t +cpu_signal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1) +{ + return cpu_signal_internal(target_proc, signal, p0, p1, FALSE); +} + +kern_return_t +cpu_signal_deferred(cpu_data_t *target_proc) +{ + return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE); +} + +void +cpu_signal_cancel(cpu_data_t *target_proc) +{ + /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */ + if (!(target_proc->cpu_signal & SIGPdisabled)) { + PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id); + } +} + +void +cpu_signal_handler(void) +{ + cpu_signal_handler_internal(FALSE); +} + +void +cpu_signal_handler_internal(boolean_t disable_signal) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + unsigned int cpu_signal; + + + cpu_data_ptr->cpu_stat.ipi_cnt++; + cpu_data_ptr->cpu_stat.ipi_cnt_wake++; + + SCHED_STATS_IPI(current_processor()); + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + + if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) + (void)hw_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled); + else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdisabled); + + while (cpu_signal & ~SIGPdisabled) { + if (cpu_signal & SIGPdec) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdec); + rtclock_intr(FALSE); + } + if (cpu_signal & SIGPchud) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPchud); + chudxnu_cpu_signal_handler(); + } +#if KPERF + if (cpu_signal & SIGPkptimer) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPkptimer); + kperf_signal_handler((unsigned int)cpu_data_ptr->cpu_number); + } +#endif + if (cpu_signal & SIGPxcall) { + cpu_handle_xcall(cpu_data_ptr); + } + if (cpu_signal & SIGPast) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPast); + ast_check(cpu_data_ptr->cpu_processor); + } + if (cpu_signal & SIGPdebug) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdebug); + DebuggerXCall(cpu_data_ptr->cpu_int_state); + } +#if __ARM_SMP__ && defined(ARMA7) + if (cpu_signal & SIGPLWFlush) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWFlush); + cache_xcall_handler(LWFlush); + } + if (cpu_signal & SIGPLWClean) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWClean); + cache_xcall_handler(LWClean); + } +#endif + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + } +} + +void +cpu_exit_wait(int cpu) +{ + if ( cpu != master_cpu) { + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {}; + } +} + +void +cpu_machine_init(void) +{ + static boolean_t started = FALSE; + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + platform_cache_init(); + PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started); + cpu_data_ptr->cpu_flags |= StartedState; + ml_init_interrupt(); +} + +processor_t +cpu_processor_alloc(boolean_t is_boot_cpu) +{ + processor_t proc; + + if (is_boot_cpu) + return &BootProcessor; + + proc = kalloc(sizeof(*proc)); + if (!proc) + return NULL; + + bzero((void *) proc, sizeof(*proc)); + return proc; +} + +void +cpu_processor_free(processor_t proc) +{ + if (proc != NULL && proc != &BootProcessor) + kfree((void *) proc, sizeof(*proc)); +} + +processor_t +current_processor(void) +{ + return getCpuDatap()->cpu_processor; +} + +processor_t +cpu_to_processor(int cpu) +{ + cpu_data_t *cpu_data = cpu_datap(cpu); + if (cpu_data != NULL) + return cpu_data->cpu_processor; + else + return NULL; +} + +cpu_data_t * +processor_to_cpu_datap(processor_t processor) +{ + cpu_data_t *target_cpu_datap; + + assert(processor->cpu_id < MAX_CPUS); + assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL); + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[processor->cpu_id].cpu_data_vaddr; + assert(target_cpu_datap->cpu_processor == processor); + + return target_cpu_datap; +} + +ast_t * +ast_pending(void) +{ + return (&getCpuDatap()->cpu_pending_ast); +} + +cpu_type_t +slot_type(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_type); +} + +cpu_subtype_t +slot_subtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_subtype); +} + +cpu_threadtype_t +slot_threadtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_threadtype); +} + +cpu_type_t +cpu_type(void) +{ + return (getCpuDatap()->cpu_type); +} + +cpu_subtype_t +cpu_subtype(void) +{ + return (getCpuDatap()->cpu_subtype); +} + +cpu_threadtype_t +cpu_threadtype(void) +{ + return (getCpuDatap()->cpu_threadtype); +} + +int +cpu_number(void) +{ + return (getCpuDatap()->cpu_number); +} + +uint64_t +ml_get_wake_timebase(void) +{ + return wake_abstime; +} +