+void
+mp_rendezvous_break_lock(void)
+{
+ simple_lock_init(&mp_rv_lock, 0);
+}
+
+static void
+setup_disable_intrs(__unused void * param_not_used)
+{
+ /* disable interrupts before the first barrier */
+ boolean_t intr = ml_set_interrupts_enabled(FALSE);
+
+ current_cpu_datap()->cpu_iflag = intr;
+ DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
+}
+
+static void
+teardown_restore_intrs(__unused void * param_not_used)
+{
+ /* restore interrupt flag following MTRR changes */
+ ml_set_interrupts_enabled(current_cpu_datap()->cpu_iflag);
+ DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
+}
+
+/*
+ * A wrapper to mp_rendezvous() to call action_func() with interrupts disabled.
+ * This is exported for use by kexts.
+ */
+void
+mp_rendezvous_no_intrs(
+ void (*action_func)(void *),
+ void *arg)
+{
+ mp_rendezvous(setup_disable_intrs,
+ action_func,
+ teardown_restore_intrs,
+ arg);
+}
+
+
+typedef struct {
+ queue_chain_t link; /* queue linkage */
+ void (*func)(void *,void *); /* routine to call */
+ void *arg0; /* routine's 1st arg */
+ void *arg1; /* routine's 2nd arg */
+ volatile long *countp; /* completion counter */
+} mp_call_t;
+
+#define MP_CPUS_CALL_BUFS_PER_CPU MAX_CPUS
+static queue_head_t mp_cpus_call_freelist;
+static queue_head_t mp_cpus_call_queue[MAX_CPUS];
+/*
+ * The free list and the per-cpu call queues are protected by the following
+ * lock which is taken wil interrupts disabled.
+ */
+decl_simple_lock_data(,mp_cpus_call_lock);
+
+static inline boolean_t
+mp_call_lock(void)
+{
+ boolean_t intrs_enabled;
+
+ intrs_enabled = ml_set_interrupts_enabled(FALSE);
+ simple_lock(&mp_cpus_call_lock);
+
+ return intrs_enabled;
+}
+
+static inline boolean_t
+mp_call_is_locked(void)
+{
+ return !ml_get_interrupts_enabled() &&
+ hw_lock_held((hw_lock_t)&mp_cpus_call_lock);
+}
+
+static inline void
+mp_call_unlock(boolean_t intrs_enabled)
+{
+ simple_unlock(&mp_cpus_call_lock);
+ ml_set_interrupts_enabled(intrs_enabled);
+}
+
+static inline mp_call_t *
+mp_call_alloc(void)
+{
+ mp_call_t *callp;
+
+ assert(mp_call_is_locked());
+ if (queue_empty(&mp_cpus_call_freelist))
+ return NULL;
+ queue_remove_first(&mp_cpus_call_freelist, callp, typeof(callp), link);
+ return callp;
+}
+
+static inline void
+mp_call_free(mp_call_t *callp)
+{
+ assert(mp_call_is_locked());
+ queue_enter_first(&mp_cpus_call_freelist, callp, typeof(callp), link);
+}
+
+static inline mp_call_t *
+mp_call_dequeue(queue_t call_queue)
+{
+ mp_call_t *callp;
+
+ assert(mp_call_is_locked());
+ if (queue_empty(call_queue))
+ return NULL;
+ queue_remove_first(call_queue, callp, typeof(callp), link);
+ return callp;
+}
+
+/* Called on the boot processor to initialize global structures */
+static void
+mp_cpus_call_init(void)
+{
+ DBG("mp_cpus_call_init()\n");
+ simple_lock_init(&mp_cpus_call_lock, 0);
+ queue_init(&mp_cpus_call_freelist);
+}
+
+/*
+ * Called by each processor to add call buffers to the free list
+ * and to initialize the per-cpu call queue.
+ * Also called but ignored on slave processors on re-start/wake.
+ */
+static void
+mp_cpus_call_cpu_init(void)
+{
+ boolean_t intrs_enabled;
+ int i;
+ mp_call_t *callp;
+
+ if (mp_cpus_call_queue[cpu_number()].next != NULL)
+ return; /* restart/wake case: called already */
+
+ queue_init(&mp_cpus_call_queue[cpu_number()]);
+ for (i = 0; i < MP_CPUS_CALL_BUFS_PER_CPU; i++) {
+ callp = (mp_call_t *) kalloc(sizeof(mp_call_t));
+ intrs_enabled = mp_call_lock();
+ mp_call_free(callp);
+ mp_call_unlock(intrs_enabled);
+ }
+
+ DBG("mp_cpus_call_init() done on cpu %d\n", cpu_number());
+}
+
+/*
+ * This is called from cpu_signal_handler() to process an MP_CALL signal.
+ * And also from i386_deactivate_cpu() when a cpu is being taken offline.
+ */
+static void
+mp_cpus_call_action(void)
+{
+ queue_t cpu_head;
+ boolean_t intrs_enabled;
+ mp_call_t *callp;
+ mp_call_t call;
+
+ assert(!ml_get_interrupts_enabled());
+ cpu_head = &mp_cpus_call_queue[cpu_number()];
+ intrs_enabled = mp_call_lock();
+ while ((callp = mp_call_dequeue(cpu_head)) != NULL) {
+ /* Copy call request to the stack to free buffer */
+ call = *callp;
+ mp_call_free(callp);
+ if (call.func != NULL) {
+ mp_call_unlock(intrs_enabled);
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL_ACTION,
+ call.func, call.arg0, call.arg1, call.countp, 0);
+ call.func(call.arg0, call.arg1);
+ (void) mp_call_lock();
+ }
+ if (call.countp != NULL)
+ atomic_incl(call.countp, 1);
+ }
+ mp_call_unlock(intrs_enabled);
+}
+
+static boolean_t
+mp_call_queue(
+ int cpu,
+ void (*action_func)(void *, void *),
+ void *arg0,
+ void *arg1,
+ volatile long *countp)
+{
+ queue_t cpu_head = &mp_cpus_call_queue[cpu];
+ mp_call_t *callp;
+
+ assert(mp_call_is_locked());
+ callp = mp_call_alloc();
+ if (callp == NULL)
+ return FALSE;
+
+ callp->func = action_func;
+ callp->arg0 = arg0;
+ callp->arg1 = arg1;
+ callp->countp = countp;
+
+ queue_enter(cpu_head, callp, typeof(callp), link);
+
+ return TRUE;
+}
+
+/*
+ * mp_cpus_call() runs a given function on cpus specified in a given cpu mask.
+ * Possible modes are:
+ * SYNC: function is called serially on target cpus in logical cpu order
+ * waiting for each call to be acknowledged before proceeding
+ * ASYNC: function call is queued to the specified cpus
+ * waiting for all calls to complete in parallel before returning
+ * NOSYNC: function calls are queued
+ * but we return before confirmation of calls completing.
+ * The action function may be NULL.
+ * The cpu mask may include the local cpu. Offline cpus are ignored.
+ * The return value is the number of cpus on which the call was made or queued.
+ */
+cpu_t
+mp_cpus_call(
+ cpumask_t cpus,
+ mp_sync_t mode,
+ void (*action_func)(void *),
+ void *arg)
+{
+ return mp_cpus_call1(
+ cpus,
+ mode,
+ (void (*)(void *,void *))action_func,
+ arg,
+ NULL,
+ NULL,
+ NULL);
+}
+
+static void
+mp_cpus_call_wait(boolean_t intrs_enabled,
+ long mp_cpus_signals,
+ volatile long *mp_cpus_calls)
+{
+ queue_t cpu_head;
+
+ cpu_head = &mp_cpus_call_queue[cpu_number()];
+
+ while (*mp_cpus_calls < mp_cpus_signals) {
+ if (!intrs_enabled) {
+ if (!queue_empty(cpu_head))
+ mp_cpus_call_action();
+
+ handle_pending_TLB_flushes();
+ }
+ cpu_pause();
+ }
+}
+
+cpu_t
+mp_cpus_call1(
+ cpumask_t cpus,
+ mp_sync_t mode,
+ void (*action_func)(void *, void *),
+ void *arg0,
+ void *arg1,
+ cpumask_t *cpus_calledp,
+ cpumask_t *cpus_notcalledp)
+{
+ cpu_t cpu;
+ boolean_t intrs_enabled = FALSE;
+ boolean_t call_self = FALSE;
+ cpumask_t cpus_called = 0;
+ cpumask_t cpus_notcalled = 0;
+ long mp_cpus_signals = 0;
+ volatile long mp_cpus_calls = 0;
+
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL | DBG_FUNC_START,
+ cpus, mode, action_func, arg0, arg1);
+
+ if (!smp_initialized) {
+ if ((cpus & CPUMASK_SELF) == 0)
+ goto out;
+ if (action_func != NULL) {
+ intrs_enabled = ml_set_interrupts_enabled(FALSE);
+ action_func(arg0, arg1);
+ ml_set_interrupts_enabled(intrs_enabled);
+ }
+ call_self = TRUE;
+ goto out;
+ }
+
+ /*
+ * Queue the call for each non-local requested cpu.
+ * The topo lock is not taken. Instead we sniff the cpu_running state
+ * and then re-check it after taking the call lock. A cpu being taken
+ * offline runs the action function after clearing the cpu_running.
+ */
+ for (cpu = 0; cpu < (cpu_t) real_ncpus; cpu++) {
+ if (((cpu_to_cpumask(cpu) & cpus) == 0) ||
+ !cpu_datap(cpu)->cpu_running)
+ continue;
+ if (cpu == (cpu_t) cpu_number()) {
+ /*
+ * We don't IPI ourself and if calling asynchronously,
+ * we defer our call until we have signalled all others.
+ */
+ call_self = TRUE;
+ cpus_called |= cpu_to_cpumask(cpu);
+ if (mode == SYNC && action_func != NULL) {
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL_LOCAL,
+ action_func, arg0, arg1, 0, 0);
+ action_func(arg0, arg1);
+ }
+ } else {
+ /*
+ * Here to queue a call to cpu and IPI.
+ * Spinning for request buffer unless NOSYNC.
+ */
+ queue_call:
+ intrs_enabled = mp_call_lock();
+ if (!cpu_datap(cpu)->cpu_running) {
+ mp_call_unlock(intrs_enabled);
+ continue;
+ }
+ if (mode == NOSYNC) {
+ if (!mp_call_queue(cpu, action_func, arg0, arg1,
+ NULL)) {
+ cpus_notcalled |= cpu_to_cpumask(cpu);
+ mp_call_unlock(intrs_enabled);
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL_NOBUF,
+ cpu, 0, 0, 0, 0);
+ continue;
+ }
+ } else {
+ if (!mp_call_queue(cpu, action_func, arg0, arg1,
+ &mp_cpus_calls)) {
+ mp_call_unlock(intrs_enabled);
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL_NOBUF,
+ cpu, 0, 0, 0, 0);
+ if (!intrs_enabled) {
+ mp_cpus_call_action();
+ handle_pending_TLB_flushes();
+ }
+ cpu_pause();
+ goto queue_call;
+ }
+ }
+ mp_cpus_signals++;
+ cpus_called |= cpu_to_cpumask(cpu);
+ i386_signal_cpu(cpu, MP_CALL, ASYNC);
+ mp_call_unlock(intrs_enabled);
+ if (mode == SYNC) {
+ mp_cpus_call_wait(intrs_enabled, mp_cpus_signals, &mp_cpus_calls);
+ }
+ }
+ }
+
+ /* Call locally if mode not SYNC */
+ if (mode != SYNC && call_self ) {
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL_LOCAL,
+ action_func, arg0, arg1, 0, 0);
+ if (action_func != NULL) {
+ ml_set_interrupts_enabled(FALSE);
+ action_func(arg0, arg1);
+ ml_set_interrupts_enabled(intrs_enabled);
+ }
+ }
+
+ /* For ASYNC, now wait for all signaled cpus to complete their calls */
+ if (mode == ASYNC) {
+ mp_cpus_call_wait(intrs_enabled, mp_cpus_signals, &mp_cpus_calls);
+ }
+
+out:
+ cpu = (cpu_t) mp_cpus_signals + (call_self ? 1 : 0);
+
+ if (cpus_calledp)
+ *cpus_calledp = cpus_called;
+ if (cpus_notcalledp)
+ *cpus_notcalledp = cpus_notcalled;
+
+ KERNEL_DEBUG_CONSTANT(
+ TRACE_MP_CPUS_CALL | DBG_FUNC_END,
+ cpu, cpus_called, cpus_notcalled, 0, 0);
+
+ return cpu;
+}
+
+
+static void
+mp_broadcast_action(void)
+{
+ /* call action function */
+ if (mp_bc_action_func != NULL)
+ mp_bc_action_func(mp_bc_func_arg);
+
+ /* if we're the last one through, wake up the instigator */
+ if (atomic_decl_and_test(&mp_bc_count, 1))
+ thread_wakeup(((event_t)(uintptr_t) &mp_bc_count));
+}
+
+/*
+ * mp_broadcast() runs a given function on all active cpus.
+ * The caller blocks until the functions has run on all cpus.
+ * The caller will also block if there is another pending braodcast.
+ */
+void
+mp_broadcast(
+ void (*action_func)(void *),
+ void *arg)
+{
+ if (!smp_initialized) {
+ if (action_func != NULL)
+ action_func(arg);
+ return;
+ }
+
+ /* obtain broadcast lock */
+ lck_mtx_lock(&mp_bc_lock);
+
+ /* set static function pointers */
+ mp_bc_action_func = action_func;
+ mp_bc_func_arg = arg;
+
+ assert_wait((event_t)(uintptr_t)&mp_bc_count, THREAD_UNINT);
+
+ /*
+ * signal other processors, which will call mp_broadcast_action()
+ */
+ simple_lock(&x86_topo_lock);
+ mp_bc_ncpus = i386_active_cpus(); /* total including this cpu */
+ mp_bc_count = mp_bc_ncpus;
+ i386_signal_cpus(MP_BROADCAST, ASYNC);
+
+ /* call executor function on this cpu */
+ mp_broadcast_action();
+ simple_unlock(&x86_topo_lock);
+
+ /* block for all cpus to have run action_func */
+ if (mp_bc_ncpus > 1)
+ thread_block(THREAD_CONTINUE_NULL);
+ else
+ clear_wait(current_thread(), THREAD_AWAKENED);
+
+ /* release lock */
+ lck_mtx_unlock(&mp_bc_lock);
+}
+
+void
+i386_activate_cpu(void)
+{
+ cpu_data_t *cdp = current_cpu_datap();
+
+ assert(!ml_get_interrupts_enabled());
+
+ if (!smp_initialized) {
+ cdp->cpu_running = TRUE;
+ return;
+ }
+
+ simple_lock(&x86_topo_lock);
+ cdp->cpu_running = TRUE;
+ started_cpu();
+ simple_unlock(&x86_topo_lock);
+ flush_tlb_raw();
+}
+
+extern void etimer_timer_expire(void *arg);
+
+void
+i386_deactivate_cpu(void)
+{
+ cpu_data_t *cdp = current_cpu_datap();
+
+ assert(!ml_get_interrupts_enabled());
+
+ simple_lock(&x86_topo_lock);
+ cdp->cpu_running = FALSE;
+ simple_unlock(&x86_topo_lock);
+
+ timer_queue_shutdown(&cdp->rtclock_timer.queue);
+ cdp->rtclock_timer.deadline = EndOfAllTime;
+ mp_cpus_call(cpu_to_cpumask(master_cpu), ASYNC, etimer_timer_expire, NULL);
+
+ /*
+ * In case a rendezvous/braodcast/call was initiated to this cpu
+ * before we cleared cpu_running, we must perform any actions due.
+ */
+ if (i_bit(MP_RENDEZVOUS, &cdp->cpu_signals))
+ mp_rendezvous_action();
+ if (i_bit(MP_BROADCAST, &cdp->cpu_signals))
+ mp_broadcast_action();
+ if (i_bit(MP_CALL, &cdp->cpu_signals))
+ mp_cpus_call_action();
+ cdp->cpu_signals = 0; /* all clear */
+}
+
+int pmsafe_debug = 1;
+
projects / apple / xnu.git / blobdiff