/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <ipc/ipc_port.h>
#include <kern/kalloc.h>
+#include <security/mac_mach_internal.h>
+
+#if defined(CONFIG_XNUPOST)
+
+#include <tests/xnupost.h>
+
+#endif /* CONFIG_XNUPOST */
+
/*
* Exported interface
*/
#include <mach/mach_host_server.h>
#include <mach/processor_set_server.h>
-/*
- * Exported variables.
- */
-struct processor_set default_pset;
+struct processor_set pset0;
+struct pset_node pset_node0;
+decl_simple_lock_data(static,pset_node_lock)
+
+queue_head_t tasks;
+queue_head_t terminated_tasks; /* To be used ONLY for stackshot. */
+queue_head_t corpse_tasks;
+int tasks_count;
+int terminated_tasks_count;
+queue_head_t threads;
+int threads_count;
+decl_lck_mtx_data(,tasks_threads_lock)
+decl_lck_mtx_data(,tasks_corpse_lock)
processor_t processor_list;
unsigned int processor_count;
static processor_t processor_list_tail;
decl_simple_lock_data(,processor_list_lock)
-processor_t master_processor;
-int master_cpu = 0;
+uint32_t processor_avail_count;
-/* Forwards */
-kern_return_t processor_set_base(
- processor_set_t pset,
- policy_t policy,
- policy_base_t base,
- boolean_t change);
+processor_t master_processor;
+int master_cpu = 0;
+boolean_t sched_stats_active = FALSE;
-kern_return_t processor_set_limit(
- processor_set_t pset,
- policy_t policy,
- policy_limit_t limit,
- boolean_t change);
+processor_t processor_array[MAX_SCHED_CPUS] = { 0 };
-kern_return_t processor_set_things(
- processor_set_t pset,
- mach_port_t **thing_list,
- mach_msg_type_number_t *count,
- int type);
+#if defined(CONFIG_XNUPOST)
+kern_return_t ipi_test(void);
+extern void arm64_ipi_test(void);
-void
-processor_bootstrap(void)
+kern_return_t
+ipi_test()
{
- simple_lock_init(&processor_list_lock, 0);
+#if __arm64__
+ processor_t p;
- master_processor = cpu_to_processor(master_cpu);
+ for (p = processor_list; p != NULL; p = p->processor_list) {
+ thread_bind(p);
+ thread_block(THREAD_CONTINUE_NULL);
+ kprintf("Running IPI test on cpu %d\n", p->cpu_id);
+ arm64_ipi_test();
+ }
+
+ /* unbind thread from specific cpu */
+ thread_bind(PROCESSOR_NULL);
+ thread_block(THREAD_CONTINUE_NULL);
- processor_init(master_processor, master_cpu);
+ T_PASS("Done running IPI tests");
+#else
+ T_PASS("Unsupported platform. Not running IPI tests");
+
+#endif /* __arm64__ */
+
+ return KERN_SUCCESS;
}
+#endif /* defined(CONFIG_XNUPOST) */
-/*
- * Initialize the given processor_set structure.
- */
void
-pset_init(
- register processor_set_t pset)
+processor_bootstrap(void)
{
- register int i;
-
- /* setup run queue */
- pset->runq.highq = IDLEPRI;
- for (i = 0; i < NRQBM; i++)
- pset->runq.bitmap[i] = 0;
- setbit(MAXPRI - IDLEPRI, pset->runq.bitmap);
- pset->runq.urgency = pset->runq.count = 0;
- for (i = 0; i < NRQS; i++)
- queue_init(&pset->runq.queues[i]);
-
- queue_init(&pset->idle_queue);
- pset->idle_count = 0;
- queue_init(&pset->active_queue);
- simple_lock_init(&pset->sched_lock, 0);
- pset->run_count = pset->share_count = 0;
- pset->mach_factor = pset->load_average = 0;
- pset->pri_shift = INT8_MAX;
- queue_init(&pset->processors);
- pset->processor_count = 0;
- queue_init(&pset->tasks);
- pset->task_count = 0;
- queue_init(&pset->threads);
- pset->thread_count = 0;
- pset->ref_count = 1;
- pset->active = TRUE;
- mutex_init(&pset->lock, 0);
- pset->pset_self = IP_NULL;
- pset->pset_name_self = IP_NULL;
- pset->timeshare_quanta = 1;
+ pset_init(&pset0, &pset_node0);
+ pset_node0.psets = &pset0;
+
+ simple_lock_init(&pset_node_lock, 0);
+
+ queue_init(&tasks);
+ queue_init(&terminated_tasks);
+ queue_init(&threads);
+ queue_init(&corpse_tasks);
+
+ simple_lock_init(&processor_list_lock, 0);
+
+ master_processor = cpu_to_processor(master_cpu);
+
+ processor_init(master_processor, master_cpu, &pset0);
}
/*
- * Initialize the given processor structure for the processor in
- * the slot specified by slot_num.
+ * Initialize the given processor for the cpu
+ * indicated by cpu_id, and assign to the
+ * specified processor set.
*/
void
processor_init(
- register processor_t p,
- int slot_num)
+ processor_t processor,
+ int cpu_id,
+ processor_set_t pset)
{
- register int i;
-
- /* setup run queue */
- p->runq.highq = IDLEPRI;
- for (i = 0; i < NRQBM; i++)
- p->runq.bitmap[i] = 0;
- setbit(MAXPRI - IDLEPRI, p->runq.bitmap);
- p->runq.urgency = p->runq.count = 0;
- for (i = 0; i < NRQS; i++)
- queue_init(&p->runq.queues[i]);
-
- p->state = PROCESSOR_OFF_LINE;
- p->active_thread = p->next_thread = p->idle_thread = THREAD_NULL;
- p->processor_set = PROCESSOR_SET_NULL;
- p->current_pri = MINPRI;
- p->deadline = UINT64_MAX;
- timer_call_setup(&p->quantum_timer, thread_quantum_expire, p);
- p->timeslice = 0;
- simple_lock_init(&p->lock, 0);
- p->processor_self = IP_NULL;
- processor_data_init(p);
- PROCESSOR_DATA(p, slot_num) = slot_num;
+ spl_t s;
+
+ if (processor != master_processor) {
+ /* Scheduler state for master_processor initialized in sched_init() */
+ SCHED(processor_init)(processor);
+ }
+
+ assert(cpu_id < MAX_SCHED_CPUS);
+
+ processor->state = PROCESSOR_OFF_LINE;
+ processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL;
+ processor->processor_set = pset;
+ processor_state_update_idle(processor);
+ processor->starting_pri = MINPRI;
+ processor->cpu_id = cpu_id;
+ timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor);
+ processor->quantum_end = UINT64_MAX;
+ processor->deadline = UINT64_MAX;
+ processor->first_timeslice = FALSE;
+ processor->processor_primary = processor; /* no SMT relationship known at this point */
+ processor->processor_secondary = NULL;
+ processor->is_SMT = FALSE;
+ processor->is_recommended = (pset->recommended_bitmask & (1ULL << cpu_id)) ? TRUE : FALSE;
+ processor->processor_self = IP_NULL;
+ processor_data_init(processor);
+ processor->processor_list = NULL;
+ processor->cpu_quiesce_state = CPU_QUIESCE_COUNTER_NONE;
+ processor->cpu_quiesce_last_checkin = 0;
+
+ s = splsched();
+ pset_lock(pset);
+ bit_set(pset->cpu_bitmask, cpu_id);
+ if (pset->cpu_set_count++ == 0)
+ pset->cpu_set_low = pset->cpu_set_hi = cpu_id;
+ else {
+ pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low;
+ pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi;
+ }
+ pset_unlock(pset);
+ splx(s);
simple_lock(&processor_list_lock);
if (processor_list == NULL)
- processor_list = p;
+ processor_list = processor;
else
- processor_list_tail->processor_list = p;
- processor_list_tail = p;
+ processor_list_tail->processor_list = processor;
+ processor_list_tail = processor;
processor_count++;
- p->processor_list = NULL;
+ processor_array[cpu_id] = processor;
simple_unlock(&processor_list_lock);
}
-/*
- * pset_deallocate:
- *
- * Remove one reference to the processor set. Destroy processor_set
- * if this was the last reference.
- */
void
-pset_deallocate(
- processor_set_t pset)
+processor_set_primary(
+ processor_t processor,
+ processor_t primary)
{
- if (pset == PROCESSOR_SET_NULL)
- return;
-
- assert(pset == &default_pset);
- return;
+ assert(processor->processor_primary == primary || processor->processor_primary == processor);
+ /* Re-adjust primary point for this (possibly) secondary processor */
+ processor->processor_primary = primary;
+
+ assert(primary->processor_secondary == NULL || primary->processor_secondary == processor);
+ if (primary != processor) {
+ /* Link primary to secondary, assumes a 2-way SMT model
+ * We'll need to move to a queue if any future architecture
+ * requires otherwise.
+ */
+ assert(processor->processor_secondary == NULL);
+ primary->processor_secondary = processor;
+ /* Mark both processors as SMT siblings */
+ primary->is_SMT = TRUE;
+ processor->is_SMT = TRUE;
+
+ processor_set_t pset = processor->processor_set;
+ atomic_bit_clear(&pset->primary_map, processor->cpu_id, memory_order_relaxed);
+ }
}
-/*
- * pset_reference:
- *
- * Add one reference to the processor set.
- */
-void
-pset_reference(
- processor_set_t pset)
+processor_set_t
+processor_pset(
+ processor_t processor)
{
- if (pset == PROCESSOR_SET_NULL)
- return;
-
- assert(pset == &default_pset);
+ return (processor->processor_set);
}
-#define pset_reference_locked(pset) assert(pset == &default_pset)
-
-/*
- * pset_remove_processor() removes a processor from a processor_set.
- * It can only be called on the current processor. Caller must
- * hold lock on current processor and processor set.
- */
void
-pset_remove_processor(
- processor_set_t pset,
- processor_t processor)
+processor_state_update_idle(processor_t processor)
{
- if (pset != processor->processor_set)
- panic("pset_remove_processor: wrong pset");
-
- queue_remove(&pset->processors, processor, processor_t, processors);
- processor->processor_set = PROCESSOR_SET_NULL;
- pset->processor_count--;
- timeshare_quanta_update(pset);
+ processor->current_pri = IDLEPRI;
+ processor->current_sfi_class = SFI_CLASS_KERNEL;
+ processor->current_recommended_pset_type = PSET_SMP;
+ processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE;
}
-/*
- * pset_add_processor() adds a processor to a processor_set.
- * It can only be called on the current processor. Caller must
- * hold lock on curent processor and on pset. No reference counting on
- * processors. Processor reference to pset is implicit.
- */
void
-pset_add_processor(
- processor_set_t pset,
- processor_t processor)
+processor_state_update_from_thread(processor_t processor, thread_t thread)
{
- queue_enter(&pset->processors, processor, processor_t, processors);
- processor->processor_set = pset;
- pset->processor_count++;
- timeshare_quanta_update(pset);
+ processor->current_pri = thread->sched_pri;
+ processor->current_sfi_class = thread->sfi_class;
+ processor->current_recommended_pset_type = recommended_pset_type(thread);
+ processor->current_perfctl_class = thread_get_perfcontrol_class(thread);
}
-/*
- * pset_remove_task() removes a task from a processor_set.
- * Caller must hold locks on pset and task (unless task has
- * no references left, in which case just the pset lock is
- * needed). Pset reference count is not decremented;
- * caller must explicitly pset_deallocate.
- */
void
-pset_remove_task(
- processor_set_t pset,
- task_t task)
+processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class,
+ pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class)
{
- if (pset != task->processor_set)
- return;
-
- queue_remove(&pset->tasks, task, task_t, pset_tasks);
- pset->task_count--;
+ processor->current_pri = pri;
+ processor->current_sfi_class = sfi_class;
+ processor->current_recommended_pset_type = pset_type;
+ processor->current_perfctl_class = perfctl_class;
}
-/*
- * pset_add_task() adds a task to a processor_set.
- * Caller must hold locks on pset and task. Pset references to
- * tasks are implicit.
- */
-void
-pset_add_task(
- processor_set_t pset,
- task_t task)
+pset_node_t
+pset_node_root(void)
{
- queue_enter(&pset->tasks, task, task_t, pset_tasks);
- task->processor_set = pset;
- pset->task_count++;
- pset_reference_locked(pset);
+ return &pset_node0;
}
-/*
- * pset_remove_thread() removes a thread from a processor_set.
- * Caller must hold locks on pset and thread (but only if thread
- * has outstanding references that could be used to lookup the pset).
- * The pset reference count is not decremented; caller must explicitly
- * pset_deallocate.
- */
-void
-pset_remove_thread(
- processor_set_t pset,
- thread_t thread)
+processor_set_t
+pset_create(
+ pset_node_t node)
{
- queue_remove(&pset->threads, thread, thread_t, pset_threads);
- pset->thread_count--;
+ /* some schedulers do not support multiple psets */
+ if (SCHED(multiple_psets_enabled) == FALSE)
+ return processor_pset(master_processor);
+
+ processor_set_t *prev, pset = kalloc(sizeof (*pset));
+
+ if (pset != PROCESSOR_SET_NULL) {
+ pset_init(pset, node);
+
+ simple_lock(&pset_node_lock);
+
+ prev = &node->psets;
+ while (*prev != PROCESSOR_SET_NULL)
+ prev = &(*prev)->pset_list;
+
+ *prev = pset;
+
+ simple_unlock(&pset_node_lock);
+ }
+
+ return (pset);
}
/*
- * pset_add_thread() adds a thread to a processor_set.
- * Caller must hold locks on pset and thread. Pset references to
- * threads are implicit.
+ * Find processor set in specified node with specified cluster_id.
+ * Returns default_pset if not found.
*/
-void
-pset_add_thread(
- processor_set_t pset,
- thread_t thread)
+processor_set_t
+pset_find(
+ uint32_t cluster_id,
+ processor_set_t default_pset)
{
- queue_enter(&pset->threads, thread, thread_t, pset_threads);
- thread->processor_set = pset;
- pset->thread_count++;
- pset_reference_locked(pset);
+ simple_lock(&pset_node_lock);
+ pset_node_t node = &pset_node0;
+ processor_set_t pset = NULL;
+
+ do {
+ pset = node->psets;
+ while (pset != NULL) {
+ if (pset->pset_cluster_id == cluster_id)
+ break;
+ pset = pset->pset_list;
+ }
+ } while ((node = node->node_list) != NULL);
+ simple_unlock(&pset_node_lock);
+ if (pset == NULL)
+ return default_pset;
+ return (pset);
}
/*
- * thread_change_psets() changes the pset of a thread. Caller must
- * hold locks on both psets and thread. The old pset must be
- * explicitly pset_deallocat()'ed by caller.
+ * Initialize the given processor_set structure.
*/
void
-thread_change_psets(
- thread_t thread,
- processor_set_t old_pset,
- processor_set_t new_pset)
+pset_init(
+ processor_set_t pset,
+ pset_node_t node)
{
- queue_remove(&old_pset->threads, thread, thread_t, pset_threads);
- old_pset->thread_count--;
- queue_enter(&new_pset->threads, thread, thread_t, pset_threads);
- thread->processor_set = new_pset;
- new_pset->thread_count++;
- pset_reference_locked(new_pset);
-}
+ if (pset != &pset0) {
+ /* Scheduler state for pset0 initialized in sched_init() */
+ SCHED(pset_init)(pset);
+ SCHED(rt_init)(pset);
+ }
+ pset->online_processor_count = 0;
+ pset->load_average = 0;
+ pset->cpu_set_low = pset->cpu_set_hi = 0;
+ pset->cpu_set_count = 0;
+ pset->last_chosen = -1;
+ pset->cpu_bitmask = 0;
+ pset->recommended_bitmask = ~0ULL;
+ pset->primary_map = ~0ULL;
+ pset->cpu_state_map[PROCESSOR_OFF_LINE] = ~0ULL;
+ for (uint i = PROCESSOR_SHUTDOWN; i < PROCESSOR_STATE_LEN; i++) {
+ pset->cpu_state_map[i] = 0;
+ }
+ pset->pending_AST_cpu_mask = 0;
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+ pset->pending_deferred_AST_cpu_mask = 0;
+#endif
+ pset->pending_spill_cpu_mask = 0;
+ pset_lock_init(pset);
+ pset->pset_self = IP_NULL;
+ pset->pset_name_self = IP_NULL;
+ pset->pset_list = PROCESSOR_SET_NULL;
+ pset->node = node;
+ pset->pset_cluster_type = PSET_SMP;
+ pset->pset_cluster_id = 0;
+}
kern_return_t
processor_info_count(
kern_return_t
processor_info(
- register processor_t processor,
+ processor_t processor,
processor_flavor_t flavor,
host_t *host,
processor_info_t info,
mach_msg_type_number_t *count)
{
- register int i, slot_num, state;
+ int cpu_id, state;
kern_return_t result;
if (processor == PROCESSOR_NULL)
return (KERN_INVALID_ARGUMENT);
- slot_num = PROCESSOR_DATA(processor, slot_num);
+ cpu_id = processor->cpu_id;
switch (flavor) {
case PROCESSOR_BASIC_INFO:
{
- register processor_basic_info_t basic_info;
+ processor_basic_info_t basic_info;
if (*count < PROCESSOR_BASIC_INFO_COUNT)
return (KERN_FAILURE);
basic_info = (processor_basic_info_t) info;
- basic_info->cpu_type = slot_type(slot_num);
- basic_info->cpu_subtype = slot_subtype(slot_num);
+ basic_info->cpu_type = slot_type(cpu_id);
+ basic_info->cpu_subtype = slot_subtype(cpu_id);
state = processor->state;
if (state == PROCESSOR_OFF_LINE)
basic_info->running = FALSE;
else
basic_info->running = TRUE;
- basic_info->slot_num = slot_num;
+ basic_info->slot_num = cpu_id;
if (processor == master_processor)
basic_info->is_master = TRUE;
else
case PROCESSOR_CPU_LOAD_INFO:
{
- register processor_cpu_load_info_t cpu_load_info;
- register integer_t *cpu_ticks;
-
- if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT)
+ processor_cpu_load_info_t cpu_load_info;
+ timer_t idle_state;
+ uint64_t idle_time_snapshot1, idle_time_snapshot2;
+ uint64_t idle_time_tstamp1, idle_time_tstamp2;
+
+ /*
+ * We capture the accumulated idle time twice over
+ * the course of this function, as well as the timestamps
+ * when each were last updated. Since these are
+ * all done using non-atomic racy mechanisms, the
+ * most we can infer is whether values are stable.
+ * timer_grab() is the only function that can be
+ * used reliably on another processor's per-processor
+ * data.
+ */
+
+ if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT)
return (KERN_FAILURE);
- cpu_load_info = (processor_cpu_load_info_t) info;
- cpu_ticks = PROCESSOR_DATA(processor, cpu_ticks);
- for (i=0; i < CPU_STATE_MAX; i++)
- cpu_load_info->cpu_ticks[i] = cpu_ticks[i];
+ cpu_load_info = (processor_cpu_load_info_t) info;
+ if (precise_user_kernel_time) {
+ cpu_load_info->cpu_ticks[CPU_STATE_USER] =
+ (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval);
+ cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] =
+ (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval);
+ } else {
+ uint64_t tval = timer_grab(&PROCESSOR_DATA(processor, user_state)) +
+ timer_grab(&PROCESSOR_DATA(processor, system_state));
+
+ cpu_load_info->cpu_ticks[CPU_STATE_USER] = (uint32_t)(tval / hz_tick_interval);
+ cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0;
+ }
+
+ idle_state = &PROCESSOR_DATA(processor, idle_state);
+ idle_time_snapshot1 = timer_grab(idle_state);
+ idle_time_tstamp1 = idle_state->tstamp;
+
+ /*
+ * Idle processors are not continually updating their
+ * per-processor idle timer, so it may be extremely
+ * out of date, resulting in an over-representation
+ * of non-idle time between two measurement
+ * intervals by e.g. top(1). If we are non-idle, or
+ * have evidence that the timer is being updated
+ * concurrently, we consider its value up-to-date.
+ */
+ if (PROCESSOR_DATA(processor, current_state) != idle_state) {
+ cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+ (uint32_t)(idle_time_snapshot1 / hz_tick_interval);
+ } else if ((idle_time_snapshot1 != (idle_time_snapshot2 = timer_grab(idle_state))) ||
+ (idle_time_tstamp1 != (idle_time_tstamp2 = idle_state->tstamp))){
+ /* Idle timer is being updated concurrently, second stamp is good enough */
+ cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+ (uint32_t)(idle_time_snapshot2 / hz_tick_interval);
+ } else {
+ /*
+ * Idle timer may be very stale. Fortunately we have established
+ * that idle_time_snapshot1 and idle_time_tstamp1 are unchanging
+ */
+ idle_time_snapshot1 += mach_absolute_time() - idle_time_tstamp1;
+
+ cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+ (uint32_t)(idle_time_snapshot1 / hz_tick_interval);
+ }
+
+ cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
*count = PROCESSOR_CPU_LOAD_INFO_COUNT;
*host = &realhost;
}
default:
- result = cpu_info(flavor, slot_num, info, count);
+ result = cpu_info(flavor, cpu_id, info, count);
if (result == KERN_SUCCESS)
*host = &realhost;
kern_return_t
processor_start(
- processor_t processor)
+ processor_t processor)
{
- kern_return_t result;
- thread_t thread;
- spl_t s;
+ processor_set_t pset;
+ thread_t thread;
+ kern_return_t result;
+ spl_t s;
- if (processor == PROCESSOR_NULL)
+ if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL)
return (KERN_INVALID_ARGUMENT);
if (processor == master_processor) {
- thread_t self = current_thread();
processor_t prev;
- prev = thread_bind(self, processor);
+ prev = thread_bind(processor);
thread_block(THREAD_CONTINUE_NULL);
- result = cpu_start(PROCESSOR_DATA(processor, slot_num));
+ result = cpu_start(processor->cpu_id);
- thread_bind(self, prev);
+ thread_bind(prev);
return (result);
}
s = splsched();
- processor_lock(processor);
+ pset = processor->processor_set;
+ pset_lock(pset);
if (processor->state != PROCESSOR_OFF_LINE) {
- processor_unlock(processor);
+ pset_unlock(pset);
splx(s);
return (KERN_FAILURE);
}
- processor->state = PROCESSOR_START;
- processor_unlock(processor);
+ pset_update_processor_state(pset, processor, PROCESSOR_START);
+ pset_unlock(pset);
splx(s);
/*
result = idle_thread_create(processor);
if (result != KERN_SUCCESS) {
s = splsched();
- processor_lock(processor);
- processor->state = PROCESSOR_OFF_LINE;
- processor_unlock(processor);
+ pset_lock(pset);
+ pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
+ pset_unlock(pset);
splx(s);
return (result);
result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread);
if (result != KERN_SUCCESS) {
s = splsched();
- processor_lock(processor);
- processor->state = PROCESSOR_OFF_LINE;
- processor_unlock(processor);
+ pset_lock(pset);
+ pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
+ pset_unlock(pset);
splx(s);
return (result);
thread->bound_processor = processor;
processor->next_thread = thread;
thread->state = TH_RUN;
+ thread->last_made_runnable_time = mach_absolute_time();
thread_unlock(thread);
splx(s);
if (processor->processor_self == IP_NULL)
ipc_processor_init(processor);
- result = cpu_start(PROCESSOR_DATA(processor, slot_num));
+ result = cpu_start(processor->cpu_id);
if (result != KERN_SUCCESS) {
s = splsched();
- processor_lock(processor);
- processor->state = PROCESSOR_OFF_LINE;
- timer_call_shutdown(processor);
- processor_unlock(processor);
+ pset_lock(pset);
+ pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
+ pset_unlock(pset);
splx(s);
return (result);
if (processor == PROCESSOR_NULL)
return(KERN_INVALID_ARGUMENT);
- return(cpu_control(PROCESSOR_DATA(processor, slot_num), info, count));
-}
-
-/*
- * Calculate the appropriate timesharing quanta based on set load.
- */
-
-void
-timeshare_quanta_update(
- processor_set_t pset)
-{
- int pcount = pset->processor_count;
- int i = pset->runq.count;
-
- if (i >= pcount)
- i = 1;
- else
- if (i <= 1)
- i = pcount;
- else
- i = (pcount + (i / 2)) / i;
-
- pset->timeshare_quanta = i;
+ return(cpu_control(processor->cpu_id, info, count));
}
kern_return_t
processor_t processor,
processor_set_t *pset)
{
- int state;
+ int state;
+
+ if (processor == PROCESSOR_NULL)
+ return(KERN_INVALID_ARGUMENT);
state = processor->state;
if (state == PROCESSOR_SHUTDOWN || state == PROCESSOR_OFF_LINE)
return(KERN_FAILURE);
- *pset = processor->processor_set;
- pset_reference(*pset);
+ *pset = &pset0;
+
return(KERN_SUCCESS);
}
return(KERN_INVALID_ARGUMENT);
if (flavor == PROCESSOR_SET_BASIC_INFO) {
- register processor_set_basic_info_t basic_info;
+ processor_set_basic_info_t basic_info;
if (*count < PROCESSOR_SET_BASIC_INFO_COUNT)
return(KERN_FAILURE);
basic_info = (processor_set_basic_info_t) info;
- basic_info->processor_count = pset->processor_count;
+ basic_info->processor_count = processor_avail_count;
basic_info->default_policy = POLICY_TIMESHARE;
*count = PROCESSOR_SET_BASIC_INFO_COUNT;
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_TIMESHARE_DEFAULT) {
- register policy_timeshare_base_t ts_base;
+ policy_timeshare_base_t ts_base;
if (*count < POLICY_TIMESHARE_BASE_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_FIFO_DEFAULT) {
- register policy_fifo_base_t fifo_base;
+ policy_fifo_base_t fifo_base;
if (*count < POLICY_FIFO_BASE_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_RR_DEFAULT) {
- register policy_rr_base_t rr_base;
+ policy_rr_base_t rr_base;
if (*count < POLICY_RR_BASE_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_TIMESHARE_LIMITS) {
- register policy_timeshare_limit_t ts_limit;
+ policy_timeshare_limit_t ts_limit;
if (*count < POLICY_TIMESHARE_LIMIT_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_FIFO_LIMITS) {
- register policy_fifo_limit_t fifo_limit;
+ policy_fifo_limit_t fifo_limit;
if (*count < POLICY_FIFO_LIMIT_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_RR_LIMITS) {
- register policy_rr_limit_t rr_limit;
+ policy_rr_limit_t rr_limit;
if (*count < POLICY_RR_LIMIT_COUNT)
return(KERN_FAILURE);
return(KERN_SUCCESS);
}
else if (flavor == PROCESSOR_SET_ENABLED_POLICIES) {
- register int *enabled;
+ int *enabled;
if (*count < (sizeof(*enabled)/sizeof(int)))
return(KERN_FAILURE);
processor_set_info_t info,
mach_msg_type_number_t *count)
{
- if (pset == PROCESSOR_SET_NULL)
- return (KERN_INVALID_PROCESSOR_SET);
+ if (pset == PROCESSOR_SET_NULL || pset != &pset0)
+ return (KERN_INVALID_PROCESSOR_SET);
+
+ if (flavor == PROCESSOR_SET_LOAD_INFO) {
+ processor_set_load_info_t load_info;
- if (flavor == PROCESSOR_SET_LOAD_INFO) {
- register processor_set_load_info_t load_info;
+ if (*count < PROCESSOR_SET_LOAD_INFO_COUNT)
+ return(KERN_FAILURE);
- if (*count < PROCESSOR_SET_LOAD_INFO_COUNT)
- return(KERN_FAILURE);
+ load_info = (processor_set_load_info_t) info;
- load_info = (processor_set_load_info_t) info;
+ load_info->mach_factor = sched_mach_factor;
+ load_info->load_average = sched_load_average;
- pset_lock(pset);
- load_info->task_count = pset->task_count;
- load_info->thread_count = pset->thread_count;
- load_info->mach_factor = pset->mach_factor;
- load_info->load_average = pset->load_average;
- pset_unlock(pset);
+ load_info->task_count = tasks_count;
+ load_info->thread_count = threads_count;
- *count = PROCESSOR_SET_LOAD_INFO_COUNT;
- return(KERN_SUCCESS);
- }
+ *count = PROCESSOR_SET_LOAD_INFO_COUNT;
+ return(KERN_SUCCESS);
+ }
- return(KERN_INVALID_ARGUMENT);
+ return(KERN_INVALID_ARGUMENT);
}
/*
return (KERN_INVALID_ARGUMENT);
}
-#define THING_TASK 0
-#define THING_THREAD 1
-
/*
* processor_set_things:
*
*/
kern_return_t
processor_set_things(
- processor_set_t pset,
- mach_port_t **thing_list,
- mach_msg_type_number_t *count,
- int type)
+ processor_set_t pset,
+ void **thing_list,
+ mach_msg_type_number_t *count,
+ int type)
{
- unsigned int actual; /* this many things */
- unsigned int maxthings;
unsigned int i;
+ task_t task;
+ thread_t thread;
+
+ task_t *task_list;
+ unsigned int actual_tasks;
+ vm_size_t task_size, task_size_needed;
+ thread_t *thread_list;
+ unsigned int actual_threads;
+ vm_size_t thread_size, thread_size_needed;
+
+ void *addr, *newaddr;
vm_size_t size, size_needed;
- void *addr;
- if (pset == PROCESSOR_SET_NULL)
+ if (pset == PROCESSOR_SET_NULL || pset != &pset0)
return (KERN_INVALID_ARGUMENT);
- size = 0; addr = 0;
+ task_size = 0;
+ task_size_needed = 0;
+ task_list = NULL;
+ actual_tasks = 0;
- for (;;) {
- pset_lock(pset);
- if (!pset->active) {
- pset_unlock(pset);
+ thread_size = 0;
+ thread_size_needed = 0;
+ thread_list = NULL;
+ actual_threads = 0;
- return (KERN_FAILURE);
- }
-
- if (type == THING_TASK)
- maxthings = pset->task_count;
- else
- maxthings = pset->thread_count;
+ for (;;) {
+ lck_mtx_lock(&tasks_threads_lock);
/* do we have the memory we need? */
+ if (type == PSET_THING_THREAD)
+ thread_size_needed = threads_count * sizeof(void *);
+#if !CONFIG_MACF
+ else
+#endif
+ task_size_needed = tasks_count * sizeof(void *);
- size_needed = maxthings * sizeof (mach_port_t);
- if (size_needed <= size)
+ if (task_size_needed <= task_size &&
+ thread_size_needed <= thread_size)
break;
- /* unlock the pset and allocate more memory */
- pset_unlock(pset);
-
- if (size != 0)
- kfree(addr, size);
+ /* unlock and allocate more memory */
+ lck_mtx_unlock(&tasks_threads_lock);
- assert(size_needed > 0);
- size = size_needed;
+ /* grow task array */
+ if (task_size_needed > task_size) {
+ if (task_size != 0)
+ kfree(task_list, task_size);
- addr = kalloc(size);
- if (addr == 0)
- return (KERN_RESOURCE_SHORTAGE);
- }
+ assert(task_size_needed > 0);
+ task_size = task_size_needed;
- /* OK, have memory and the processor_set is locked & active */
+ task_list = (task_t *)kalloc(task_size);
+ if (task_list == NULL) {
+ if (thread_size != 0)
+ kfree(thread_list, thread_size);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ }
- actual = 0;
- switch (type) {
+ /* grow thread array */
+ if (thread_size_needed > thread_size) {
+ if (thread_size != 0)
+ kfree(thread_list, thread_size);
- case THING_TASK:
- {
- task_t task, *tasks = (task_t *)addr;
+ assert(thread_size_needed > 0);
+ thread_size = thread_size_needed;
- for (task = (task_t)queue_first(&pset->tasks);
- !queue_end(&pset->tasks, (queue_entry_t)task);
- task = (task_t)queue_next(&task->pset_tasks)) {
- task_reference_internal(task);
- tasks[actual++] = task;
+ thread_list = (thread_t *)kalloc(thread_size);
+ if (thread_list == 0) {
+ if (task_size != 0)
+ kfree(task_list, task_size);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
}
-
- break;
}
- case THING_THREAD:
- {
- thread_t thread, *threads = (thread_t *)addr;
-
- for (i = 0, thread = (thread_t)queue_first(&pset->threads);
- !queue_end(&pset->threads, (queue_entry_t)thread);
- thread = (thread_t)queue_next(&thread->pset_threads)) {
- thread_reference_internal(thread);
- threads[actual++] = thread;
+ /* OK, have memory and the list locked */
+
+ /* If we need it, get the thread list */
+ if (type == PSET_THING_THREAD) {
+ for (thread = (thread_t)queue_first(&threads);
+ !queue_end(&threads, (queue_entry_t)thread);
+ thread = (thread_t)queue_next(&thread->threads)) {
+#if defined(SECURE_KERNEL)
+ if (thread->task != kernel_task) {
+#endif
+ thread_reference_internal(thread);
+ thread_list[actual_threads++] = thread;
+#if defined(SECURE_KERNEL)
+ }
+#endif
}
-
- break;
}
+#if !CONFIG_MACF
+ else {
+#endif
+ /* get a list of the tasks */
+ for (task = (task_t)queue_first(&tasks);
+ !queue_end(&tasks, (queue_entry_t)task);
+ task = (task_t)queue_next(&task->tasks)) {
+#if defined(SECURE_KERNEL)
+ if (task != kernel_task) {
+#endif
+ task_reference_internal(task);
+ task_list[actual_tasks++] = task;
+#if defined(SECURE_KERNEL)
+ }
+#endif
+ }
+#if !CONFIG_MACF
}
-
- pset_unlock(pset);
+#endif
- if (actual < maxthings)
- size_needed = actual * sizeof (mach_port_t);
+ lck_mtx_unlock(&tasks_threads_lock);
- if (actual == 0) {
- /* no things, so return null pointer and deallocate memory */
- *thing_list = 0;
- *count = 0;
+#if CONFIG_MACF
+ unsigned int j, used;
- if (size != 0)
- kfree(addr, size);
+ /* for each task, make sure we are allowed to examine it */
+ for (i = used = 0; i < actual_tasks; i++) {
+ if (mac_task_check_expose_task(task_list[i])) {
+ task_deallocate(task_list[i]);
+ continue;
+ }
+ task_list[used++] = task_list[i];
}
- else {
- /* if we allocated too much, must copy */
-
- if (size_needed < size) {
- void *newaddr;
-
- newaddr = kalloc(size_needed);
- if (newaddr == 0) {
- switch (type) {
+ actual_tasks = used;
+ task_size_needed = actual_tasks * sizeof(void *);
- case THING_TASK:
- {
- task_t *tasks = (task_t *)addr;
-
- for (i = 0; i < actual; i++)
- task_deallocate(tasks[i]);
- break;
- }
+ if (type == PSET_THING_THREAD) {
- case THING_THREAD:
- {
- thread_t *threads = (thread_t *)addr;
+ /* for each thread (if any), make sure it's task is in the allowed list */
+ for (i = used = 0; i < actual_threads; i++) {
+ boolean_t found_task = FALSE;
- for (i = 0; i < actual; i++)
- thread_deallocate(threads[i]);
+ task = thread_list[i]->task;
+ for (j = 0; j < actual_tasks; j++) {
+ if (task_list[j] == task) {
+ found_task = TRUE;
break;
}
- }
-
- kfree(addr, size);
- return (KERN_RESOURCE_SHORTAGE);
}
-
- bcopy((void *) addr, (void *) newaddr, size_needed);
- kfree(addr, size);
- addr = newaddr;
+ if (found_task)
+ thread_list[used++] = thread_list[i];
+ else
+ thread_deallocate(thread_list[i]);
}
+ actual_threads = used;
+ thread_size_needed = actual_threads * sizeof(void *);
+
+ /* done with the task list */
+ for (i = 0; i < actual_tasks; i++)
+ task_deallocate(task_list[i]);
+ kfree(task_list, task_size);
+ task_size = 0;
+ actual_tasks = 0;
+ task_list = NULL;
+ }
+#endif
+
+ if (type == PSET_THING_THREAD) {
+ if (actual_threads == 0) {
+ /* no threads available to return */
+ assert(task_size == 0);
+ if (thread_size != 0)
+ kfree(thread_list, thread_size);
+ *thing_list = NULL;
+ *count = 0;
+ return KERN_SUCCESS;
+ }
+ size_needed = actual_threads * sizeof(void *);
+ size = thread_size;
+ addr = thread_list;
+ } else {
+ if (actual_tasks == 0) {
+ /* no tasks available to return */
+ assert(thread_size == 0);
+ if (task_size != 0)
+ kfree(task_list, task_size);
+ *thing_list = NULL;
+ *count = 0;
+ return KERN_SUCCESS;
+ }
+ size_needed = actual_tasks * sizeof(void *);
+ size = task_size;
+ addr = task_list;
+ }
- *thing_list = (mach_port_t *)addr;
- *count = actual;
-
- /* do the conversion that Mig should handle */
-
- switch (type) {
-
- case THING_TASK:
- {
- task_t *tasks = (task_t *)addr;
-
- for (i = 0; i < actual; i++)
- (*thing_list)[i] = convert_task_to_port(tasks[i]);
- break;
+ /* if we allocated too much, must copy */
+ if (size_needed < size) {
+ newaddr = kalloc(size_needed);
+ if (newaddr == 0) {
+ for (i = 0; i < actual_tasks; i++) {
+ if (type == PSET_THING_THREAD)
+ thread_deallocate(thread_list[i]);
+ else
+ task_deallocate(task_list[i]);
+ }
+ if (size)
+ kfree(addr, size);
+ return (KERN_RESOURCE_SHORTAGE);
}
- case THING_THREAD:
- {
- thread_t *threads = (thread_t *)addr;
+ bcopy((void *) addr, (void *) newaddr, size_needed);
+ kfree(addr, size);
- for (i = 0; i < actual; i++)
- (*thing_list)[i] = convert_thread_to_port(threads[i]);
- break;
- }
- }
+ addr = newaddr;
+ size = size_needed;
}
+ *thing_list = (void **)addr;
+ *count = (unsigned int)size / sizeof(void *);
+
return (KERN_SUCCESS);
}
task_array_t *task_list,
mach_msg_type_number_t *count)
{
- return(processor_set_things(pset, (mach_port_t **)task_list, count, THING_TASK));
+ kern_return_t ret;
+ mach_msg_type_number_t i;
+
+ ret = processor_set_things(pset, (void **)task_list, count, PSET_THING_TASK);
+ if (ret != KERN_SUCCESS)
+ return ret;
+
+ /* do the conversion that Mig should handle */
+ for (i = 0; i < *count; i++)
+ (*task_list)[i] = (task_t)convert_task_to_port((*task_list)[i]);
+ return KERN_SUCCESS;
}
/*
*
* List all threads in the processor set.
*/
+#if defined(SECURE_KERNEL)
kern_return_t
processor_set_threads(
- processor_set_t pset,
- thread_array_t *thread_list,
- mach_msg_type_number_t *count)
+ __unused processor_set_t pset,
+ __unused thread_array_t *thread_list,
+ __unused mach_msg_type_number_t *count)
{
- return(processor_set_things(pset, (mach_port_t **)thread_list, count, THING_THREAD));
+ return KERN_FAILURE;
}
-
-/*
- * processor_set_base:
- *
- * Specify per-policy base priority for a processor set. Set processor
- * set default policy to the given policy. This affects newly created
- * and assigned threads. Optionally change existing ones.
- */
+#elif defined(CONFIG_EMBEDDED)
kern_return_t
-processor_set_base(
- __unused processor_set_t pset,
- __unused policy_t policy,
- __unused policy_base_t base,
- __unused boolean_t change)
+processor_set_threads(
+ __unused processor_set_t pset,
+ __unused thread_array_t *thread_list,
+ __unused mach_msg_type_number_t *count)
{
- return (KERN_INVALID_ARGUMENT);
+ return KERN_NOT_SUPPORTED;
}
-
-/*
- * processor_set_limit:
- *
- * Specify per-policy limits for a processor set. This affects
- * newly created and assigned threads. Optionally change existing
- * ones.
- */
+#else
kern_return_t
-processor_set_limit(
- __unused processor_set_t pset,
- __unused policy_t policy,
- __unused policy_limit_t limit,
- __unused boolean_t change)
+processor_set_threads(
+ processor_set_t pset,
+ thread_array_t *thread_list,
+ mach_msg_type_number_t *count)
{
- return (KERN_POLICY_LIMIT);
+ kern_return_t ret;
+ mach_msg_type_number_t i;
+
+ ret = processor_set_things(pset, (void **)thread_list, count, PSET_THING_THREAD);
+ if (ret != KERN_SUCCESS)
+ return ret;
+
+ /* do the conversion that Mig should handle */
+ for (i = 0; i < *count; i++)
+ (*thread_list)[i] = (thread_t)convert_thread_to_port((*thread_list)[i]);
+ return KERN_SUCCESS;
}
+#endif
/*
* processor_set_policy_control
{
return (KERN_INVALID_ARGUMENT);
}
+
+#undef pset_deallocate
+void pset_deallocate(processor_set_t pset);
+void
+pset_deallocate(
+__unused processor_set_t pset)
+{
+ return;
+}
+
+#undef pset_reference
+void pset_reference(processor_set_t pset);
+void
+pset_reference(
+__unused processor_set_t pset)
+{
+ return;
+}
+
+pset_cluster_type_t
+recommended_pset_type(thread_t thread)
+{
+ (void)thread;
+ return PSET_SMP;
+}
projects / apple / xnu.git / blobdiff