xnu-7195.60.75.tar.gz

[apple/xnu.git] / osfmk / arm / cpu.c

diff --git a/osfmk/arm/cpu.c b/osfmk/arm/cpu.c
index 72e8c780048ac24c1779447851976500f99c4f1e..cce170bb05695d9b4725fbb2b41f814fe4ae484f 100644 (file)
--- a/osfmk/arm/cpu.c
+++ b/osfmk/arm/cpu.c
@@ -34,6 +34,7 @@
 #include <kern/kalloc.h>
 #include <kern/machine.h>
 #include <kern/cpu_number.h>
 #include <kern/kalloc.h>
 #include <kern/machine.h>
 #include <kern/cpu_number.h>

+#include <kern/percpu.h>

 #include <kern/thread.h>
 #include <kern/timer_queue.h>
 #include <arm/cpu_data.h>
 #include <kern/thread.h>
 #include <kern/timer_queue.h>
 #include <arm/cpu_data.h>


@@ -134,8 +135,8 @@ cpu_idle(void)
        }
        cpu_data_ptr->cpu_user_debug = NULL;
 
        }
        cpu_data_ptr->cpu_user_debug = NULL;
 

-       if (cpu_data_ptr->cpu_idle_notify) {
-               ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
+       if (cpu_data_ptr->cpu_idle_notify != NULL) {
+               cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);

        }
 
        if (cpu_data_ptr->idle_timer_notify != 0) {
        }
 
        if (cpu_data_ptr->idle_timer_notify != 0) {


@@ -148,13 +149,17 @@ cpu_idle(void)
                }
                timer_resync_deadlines();
                if (cpu_data_ptr->rtcPop != lastPop) {
                }
                timer_resync_deadlines();
                if (cpu_data_ptr->rtcPop != lastPop) {

+                       /*
+                        * Ignore the return value here: this CPU has called idle_notify and
+                        * committed to going idle.
+                        */

                        SetIdlePop();
                }
        }
 
 #if KPC
        kpc_idle();
                        SetIdlePop();
                }
        }
 
 #if KPC
        kpc_idle();

-#endif
+#endif /* KPC */

 
        platform_cache_idle_enter();
        cpu_idle_wfi((boolean_t) wfi_fast);
 
        platform_cache_idle_enter();
        cpu_idle_wfi((boolean_t) wfi_fast);


@@ -181,8 +186,8 @@ cpu_idle_exit(boolean_t from_reset __unused)
 
        pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread());
 
 
        pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread());
 

-       if (cpu_data_ptr->cpu_idle_notify) {
-               ((processor_idle_t) cpu_data_ptr->cpu_idle_notify)(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
+       if (cpu_data_ptr->cpu_idle_notify != NULL) {
+               cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);

        }
 
        if (cpu_data_ptr->idle_timer_notify != 0) {
        }
 
        if (cpu_data_ptr->idle_timer_notify != 0) {


@@ -265,7 +270,6 @@ cpu_init(void)
        }
        cdp->cpu_stat.irq_ex_cnt_wake = 0;
        cdp->cpu_stat.ipi_cnt_wake = 0;
        }
        cdp->cpu_stat.irq_ex_cnt_wake = 0;
        cdp->cpu_stat.ipi_cnt_wake = 0;

-       cdp->cpu_stat.timer_cnt_wake = 0;

        cdp->cpu_running = TRUE;
        cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
        cdp->cpu_sleep_token = 0x0UL;
        cdp->cpu_running = TRUE;
        cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
        cdp->cpu_sleep_token = 0x0UL;


@@ -309,37 +313,34 @@ cpu_data_free(cpu_data_t *cpu_data_ptr)
                return;
        }
 
                return;
        }
 

-       cpu_processor_free( cpu_data_ptr->cpu_processor);
-       if (CpuDataEntries[cpu_data_ptr->cpu_number].cpu_data_vaddr == cpu_data_ptr) {
+       int cpu_number = cpu_data_ptr->cpu_number;
+
+       if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {

                OSDecrementAtomic((SInt32*)&real_ncpus);
                OSDecrementAtomic((SInt32*)&real_ncpus);

-               CpuDataEntries[cpu_data_ptr->cpu_number].cpu_data_vaddr = NULL;
-               CpuDataEntries[cpu_data_ptr->cpu_number].cpu_data_paddr = 0;
+               CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
+               CpuDataEntries[cpu_number].cpu_data_paddr = 0;

                __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
        }
        (kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
        (kfree)((void *)(cpu_data_ptr->fiqstack_top - FIQSTACK_SIZE), FIQSTACK_SIZE);
                __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
        }
        (kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
        (kfree)((void *)(cpu_data_ptr->fiqstack_top - FIQSTACK_SIZE), FIQSTACK_SIZE);

-       kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t));

 }
 
 void
 cpu_data_init(cpu_data_t *cpu_data_ptr)
 {
 }
 
 void
 cpu_data_init(cpu_data_t *cpu_data_ptr)
 {

-       uint32_t i = 0;
-

        cpu_data_ptr->cpu_flags = 0;
 #if     __arm__
        cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 #endif
        cpu_data_ptr->cpu_flags = 0;
 #if     __arm__
        cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable;
 #endif

-       cpu_data_ptr->interrupts_enabled = 0;

        cpu_data_ptr->cpu_int_state = 0;
        cpu_data_ptr->cpu_pending_ast = AST_NONE;
        cpu_data_ptr->cpu_int_state = 0;
        cpu_data_ptr->cpu_pending_ast = AST_NONE;

-       cpu_data_ptr->cpu_cache_dispatch = (void *) 0;
+       cpu_data_ptr->cpu_cache_dispatch = NULL;

        cpu_data_ptr->rtcPop = EndOfAllTime;
        cpu_data_ptr->rtclock_datap = &RTClockData;
        cpu_data_ptr->cpu_user_debug = NULL;
        cpu_data_ptr->cpu_base_timebase_low = 0;
        cpu_data_ptr->cpu_base_timebase_high = 0;
        cpu_data_ptr->rtcPop = EndOfAllTime;
        cpu_data_ptr->rtclock_datap = &RTClockData;
        cpu_data_ptr->cpu_user_debug = NULL;
        cpu_data_ptr->cpu_base_timebase_low = 0;
        cpu_data_ptr->cpu_base_timebase_high = 0;

-       cpu_data_ptr->cpu_idle_notify = (void *) 0;
+       cpu_data_ptr->cpu_idle_notify = NULL;

        cpu_data_ptr->cpu_idle_latency = 0x0ULL;
        cpu_data_ptr->cpu_idle_pop = 0x0ULL;
        cpu_data_ptr->cpu_reset_type = 0x0UL;
        cpu_data_ptr->cpu_idle_latency = 0x0ULL;
        cpu_data_ptr->cpu_idle_pop = 0x0ULL;
        cpu_data_ptr->cpu_reset_type = 0x0UL;


@@ -369,7 +370,7 @@ cpu_data_init(cpu_data_t *cpu_data_ptr)
        cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
        cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
 
        cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
        cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
 

-#if     __ARM_SMP__ && defined(ARMA7)
+#if     defined(ARMA7)

        cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL;
        cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL;
        cpu_data_ptr->cpu_CLWClean_req = 0x0ULL;
        cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL;
        cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL;
        cpu_data_ptr->cpu_CLWClean_req = 0x0ULL;


@@ -377,15 +378,15 @@ cpu_data_init(cpu_data_t *cpu_data_ptr)
        cpu_data_ptr->cpu_CLW_active = 0x1UL;
 #endif
 
        cpu_data_ptr->cpu_CLW_active = 0x1UL;
 #endif
 

+#if !XNU_MONITOR

        pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
 
        pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL;
        pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0;
        pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
 
        pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
 
        pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL;
        pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0;
        pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
 

-       for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) {
-               pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0;
-       }
+       bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
+#endif

        cpu_data_ptr->halt_status = CPU_NOT_HALTED;
 }
 
        cpu_data_ptr->halt_status = CPU_NOT_HALTED;
 }
 


@@ -395,7 +396,7 @@ cpu_data_register(cpu_data_t *cpu_data_ptr)
        int cpu;
 
        cpu = OSIncrementAtomic((SInt32*)&real_ncpus);
        int cpu;
 
        cpu = OSIncrementAtomic((SInt32*)&real_ncpus);

-       if (real_ncpus > MAX_CPUS) {
+       if (real_ncpus > ml_get_cpu_count()) {

                return KERN_FAILURE;
        }
 
                return KERN_FAILURE;
        }
 


@@ -414,30 +415,32 @@ cpu_start(int cpu)
                cpu_machine_init();
                return KERN_SUCCESS;
        } else {
                cpu_machine_init();
                return KERN_SUCCESS;
        } else {

-#if     __ARM_SMP__

                cpu_data_t      *cpu_data_ptr;
                thread_t        first_thread;
                cpu_data_t      *cpu_data_ptr;
                thread_t        first_thread;

+               processor_t     processor;

 
                cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
                cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
 
 
                cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
                cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
 

+#if !XNU_MONITOR

                cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL;
                cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL;

+#endif

 
 

-               if (cpu_data_ptr->cpu_processor->startup_thread != THREAD_NULL) {
-                       first_thread = cpu_data_ptr->cpu_processor->startup_thread;
+               processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
+               if (processor->startup_thread != THREAD_NULL) {
+                       first_thread = processor->startup_thread;

                } else {
                } else {

-                       first_thread = cpu_data_ptr->cpu_processor->idle_thread;
+                       first_thread = processor->idle_thread;

                }
                cpu_data_ptr->cpu_active_thread = first_thread;
                first_thread->machine.CpuDatap = cpu_data_ptr;
                }
                cpu_data_ptr->cpu_active_thread = first_thread;
                first_thread->machine.CpuDatap = cpu_data_ptr;

+               first_thread->machine.pcpu_data_base =
+                   (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);

 
                flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
                flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
                (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
                return KERN_SUCCESS;
 
                flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
                flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
                (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
                return KERN_SUCCESS;

-#else
-               return KERN_FAILURE;
-#endif

        }
 }
 
        }
 }
 


@@ -478,7 +481,8 @@ ml_arm_sleep(void)
                cpu_data_t      *target_cdp;
                unsigned int    cpu;
 
                cpu_data_t      *target_cdp;
                unsigned int    cpu;
 

-               for (cpu = 0; cpu < MAX_CPUS; cpu++) {
+               const unsigned int max_cpu_id = ml_get_max_cpu_number();
+               for (cpu = 0; cpu <= max_cpu_id; cpu++) {

                        target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
                        if (target_cdp == (cpu_data_t *)NULL) {
                                break;
                        target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
                        if (target_cdp == (cpu_data_t *)NULL) {
                                break;


@@ -502,7 +506,7 @@ ml_arm_sleep(void)
                CleanPoU_Dcache();
        }
        cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
                CleanPoU_Dcache();
        }
        cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;

-#if     __ARM_SMP__ && defined(ARMA7)
+#if     defined(ARMA7)

        cpu_data_ptr->cpu_CLWFlush_req = 0;
        cpu_data_ptr->cpu_CLWClean_req = 0;
        __builtin_arm_dmb(DMB_ISH);
        cpu_data_ptr->cpu_CLWFlush_req = 0;
        cpu_data_ptr->cpu_CLWClean_req = 0;
        __builtin_arm_dmb(DMB_ISH);